2024-11-21 00:15:06,496 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 00:15:06,548 main DEBUG Took 0.025558 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 00:15:06,549 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 00:15:06,549 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 00:15:06,551 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 00:15:06,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,563 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 00:15:06,583 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,586 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,587 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,591 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,593 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,594 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,595 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,597 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,598 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,600 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,601 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:15:06,603 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,603 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 00:15:06,605 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:15:06,608 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 00:15:06,612 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 00:15:06,613 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 00:15:06,615 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 00:15:06,616 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 00:15:06,627 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 00:15:06,630 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 00:15:06,633 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 00:15:06,633 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 00:15:06,634 main DEBUG createAppenders(={Console}) 2024-11-21 00:15:06,635 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-21 00:15:06,635 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 00:15:06,636 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-21 00:15:06,637 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 00:15:06,637 main DEBUG OutputStream closed 2024-11-21 00:15:06,637 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 00:15:06,638 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 00:15:06,638 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-21 00:15:06,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 00:15:06,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 00:15:06,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 00:15:06,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 00:15:06,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 00:15:06,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 00:15:06,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 00:15:06,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 00:15:06,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 00:15:06,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 00:15:06,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 00:15:06,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 00:15:06,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 00:15:06,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 00:15:06,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 00:15:06,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 00:15:06,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 00:15:06,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 00:15:06,783 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 00:15:06,783 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-21 00:15:06,784 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 00:15:06,785 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-21T00:15:07,122 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4 2024-11-21 00:15:07,132 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 00:15:07,133 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T00:15:07,144 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-21T00:15:07,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T00:15:07,203 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2, deleteOnExit=true 2024-11-21T00:15:07,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T00:15:07,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/test.cache.data in system properties and HBase conf 2024-11-21T00:15:07,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T00:15:07,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.log.dir in system properties and HBase conf 2024-11-21T00:15:07,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T00:15:07,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T00:15:07,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T00:15:07,340 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T00:15:07,455 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T00:15:07,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T00:15:07,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T00:15:07,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T00:15:07,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T00:15:07,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T00:15:07,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T00:15:07,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T00:15:07,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T00:15:07,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T00:15:07,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/nfs.dump.dir in system properties and HBase conf 2024-11-21T00:15:07,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/java.io.tmpdir in system properties and HBase conf 2024-11-21T00:15:07,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T00:15:07,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T00:15:07,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T00:15:08,885 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T00:15:08,990 INFO [Time-limited test {}] log.Log(170): Logging initialized @3663ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T00:15:09,073 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:15:09,141 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T00:15:09,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T00:15:09,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T00:15:09,167 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T00:15:09,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:15:09,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.log.dir/,AVAILABLE} 2024-11-21T00:15:09,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T00:15:09,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/java.io.tmpdir/jetty-localhost-33547-hadoop-hdfs-3_4_1-tests_jar-_-any-6186955150366016666/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T00:15:09,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33547} 2024-11-21T00:15:09,407 INFO [Time-limited test {}] server.Server(415): Started @4081ms 2024-11-21T00:15:10,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:15:10,248 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T00:15:10,251 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T00:15:10,251 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T00:15:10,251 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T00:15:10,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.log.dir/,AVAILABLE} 2024-11-21T00:15:10,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T00:15:10,390 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/java.io.tmpdir/jetty-localhost-42475-hadoop-hdfs-3_4_1-tests_jar-_-any-1928098871898346708/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T00:15:10,391 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:42475} 2024-11-21T00:15:10,391 INFO [Time-limited test {}] server.Server(415): Started @5066ms 2024-11-21T00:15:10,447 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T00:15:11,933 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data1/current/BP-1098900603-172.17.0.2-1732148108301/current, will proceed with Du for space computation calculation, 2024-11-21T00:15:11,933 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data2/current/BP-1098900603-172.17.0.2-1732148108301/current, will proceed with Du for space computation calculation, 2024-11-21T00:15:11,962 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T00:15:12,012 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b94ef8d1dc49014 with lease ID 0xfb6378c565c90926: Processing first storage report for DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a from datanode DatanodeRegistration(127.0.0.1:45765, datanodeUuid=7e3c78df-4fbd-461e-beb8-621a666ccb9b, infoPort=38555, infoSecurePort=0, ipcPort=43823, storageInfo=lv=-57;cid=testClusterID;nsid=1942582819;c=1732148108301) 2024-11-21T00:15:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b94ef8d1dc49014 with lease ID 0xfb6378c565c90926: from storage DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a node DatanodeRegistration(127.0.0.1:45765, datanodeUuid=7e3c78df-4fbd-461e-beb8-621a666ccb9b, infoPort=38555, infoSecurePort=0, ipcPort=43823, storageInfo=lv=-57;cid=testClusterID;nsid=1942582819;c=1732148108301), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T00:15:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b94ef8d1dc49014 with lease ID 0xfb6378c565c90926: Processing first storage report for DS-0628cbd9-d7d8-4130-8fad-445e89bb854a from datanode DatanodeRegistration(127.0.0.1:45765, datanodeUuid=7e3c78df-4fbd-461e-beb8-621a666ccb9b, infoPort=38555, infoSecurePort=0, ipcPort=43823, storageInfo=lv=-57;cid=testClusterID;nsid=1942582819;c=1732148108301) 2024-11-21T00:15:12,015 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b94ef8d1dc49014 with lease ID 0xfb6378c565c90926: from storage DS-0628cbd9-d7d8-4130-8fad-445e89bb854a node DatanodeRegistration(127.0.0.1:45765, datanodeUuid=7e3c78df-4fbd-461e-beb8-621a666ccb9b, infoPort=38555, infoSecurePort=0, ipcPort=43823, storageInfo=lv=-57;cid=testClusterID;nsid=1942582819;c=1732148108301), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T00:15:12,096 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4 2024-11-21T00:15:12,172 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/zookeeper_0, clientPort=54550, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T00:15:12,183 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54550 2024-11-21T00:15:12,196 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:12,199 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:12,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741825_1001 (size=7) 2024-11-21T00:15:12,819 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b with version=8 2024-11-21T00:15:12,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/hbase-staging 2024-11-21T00:15:12,916 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T00:15:13,190 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5ed4808ef0e6:0 server-side Connection retries=45 2024-11-21T00:15:13,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,204 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T00:15:13,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T00:15:13,362 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T00:15:13,418 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T00:15:13,428 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T00:15:13,432 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T00:15:13,456 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5019 (auto-detected) 2024-11-21T00:15:13,457 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T00:15:13,474 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39691 2024-11-21T00:15:13,496 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39691 connecting to ZooKeeper ensemble=127.0.0.1:54550 2024-11-21T00:15:13,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396910x0, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T00:15:13,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39691-0x1015abea6c30000 connected 2024-11-21T00:15:13,745 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:13,748 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:13,761 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:15:13,765 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b, hbase.cluster.distributed=false 2024-11-21T00:15:13,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T00:15:13,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39691 2024-11-21T00:15:13,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39691 2024-11-21T00:15:13,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39691 2024-11-21T00:15:13,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39691 2024-11-21T00:15:13,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39691 2024-11-21T00:15:13,902 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5ed4808ef0e6:0 server-side Connection retries=45 2024-11-21T00:15:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,904 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T00:15:13,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:15:13,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T00:15:13,908 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T00:15:13,912 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T00:15:13,913 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40705 2024-11-21T00:15:13,916 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40705 connecting to ZooKeeper ensemble=127.0.0.1:54550 2024-11-21T00:15:13,918 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:13,923 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:13,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407050x0, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T00:15:13,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407050x0, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:15:13,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40705-0x1015abea6c30001 connected 2024-11-21T00:15:13,954 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T00:15:13,967 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T00:15:13,970 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T00:15:13,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T00:15:13,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40705 2024-11-21T00:15:13,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40705 2024-11-21T00:15:13,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40705 2024-11-21T00:15:13,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40705 2024-11-21T00:15:13,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40705 2024-11-21T00:15:14,006 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5ed4808ef0e6:39691 2024-11-21T00:15:14,007 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:14,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:15:14,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:15:14,028 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:14,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T00:15:14,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:14,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:14,068 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T00:15:14,070 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5ed4808ef0e6,39691,1732148113019 from backup master directory 2024-11-21T00:15:14,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:15:14,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:14,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:15:14,078 WARN [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T00:15:14,079 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:14,081 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T00:15:14,082 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T00:15:14,137 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/hbase.id] with ID: a60e3fdb-895f-4b66-963f-9689bc9d3dbc 2024-11-21T00:15:14,137 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/.tmp/hbase.id 2024-11-21T00:15:14,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741826_1002 (size=42) 2024-11-21T00:15:14,550 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/.tmp/hbase.id]:[hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/hbase.id] 2024-11-21T00:15:14,601 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:15:14,607 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T00:15:14,634 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-21T00:15:14,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:14,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:14,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741827_1003 (size=196) 2024-11-21T00:15:15,096 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:15:15,100 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T00:15:15,122 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:15,128 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:15:15,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741828_1004 (size=1189) 2024-11-21T00:15:15,249 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store 2024-11-21T00:15:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741829_1005 (size=34) 2024-11-21T00:15:15,681 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T00:15:15,684 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:15,685 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T00:15:15,686 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:15:15,686 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:15:15,687 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T00:15:15,688 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:15:15,688 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:15:15,689 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732148115685Disabling compacts and flushes for region at 1732148115685Disabling writes for close at 1732148115687 (+2 ms)Writing region close event to WAL at 1732148115688 (+1 ms)Closed at 1732148115688 2024-11-21T00:15:15,692 WARN [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/.initializing 2024-11-21T00:15:15,692 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/WALs/5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:15,704 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T00:15:15,726 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C39691%2C1732148113019, suffix=, logDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/WALs/5ed4808ef0e6,39691,1732148113019, archiveDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/oldWALs, maxLogs=10 2024-11-21T00:15:15,760 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/WALs/5ed4808ef0e6,39691,1732148113019/5ed4808ef0e6%2C39691%2C1732148113019.1732148115733, exclude list is [], retry=0 2024-11-21T00:15:15,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45765,DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a,DISK] 2024-11-21T00:15:15,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T00:15:15,841 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/WALs/5ed4808ef0e6,39691,1732148113019/5ed4808ef0e6%2C39691%2C1732148113019.1732148115733 2024-11-21T00:15:15,843 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38555:38555)] 2024-11-21T00:15:15,844 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:15:15,844 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:15,850 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:15,852 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:15,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:15,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T00:15:15,964 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:15,968 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:15,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:15,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T00:15:15,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:15,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:15,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:15,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T00:15:15,993 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:15,995 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:15,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T00:15:16,002 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:16,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:16,006 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,012 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,014 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,025 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,026 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,033 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T00:15:16,039 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:15:16,048 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:15:16,051 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59927312, jitterRate=-0.10701346397399902}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T00:15:16,060 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732148115870Initializing all the Stores at 1732148115883 (+13 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148115884 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148115885 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148115886 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148115886Cleaning up temporary data from old regions at 1732148116026 (+140 ms)Region opened successfully at 1732148116060 (+34 ms) 2024-11-21T00:15:16,062 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T00:15:16,121 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fee9eb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5ed4808ef0e6/172.17.0.2:0 2024-11-21T00:15:16,162 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T00:15:16,175 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T00:15:16,175 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T00:15:16,179 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T00:15:16,181 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T00:15:16,187 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T00:15:16,187 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T00:15:16,217 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T00:15:16,227 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T00:15:16,277 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T00:15:16,281 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T00:15:16,283 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T00:15:16,288 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T00:15:16,290 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T00:15:16,294 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T00:15:16,298 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T00:15:16,300 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T00:15:16,309 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T00:15:16,327 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T00:15:16,339 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T00:15:16,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T00:15:16,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T00:15:16,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,354 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5ed4808ef0e6,39691,1732148113019, sessionid=0x1015abea6c30000, setting cluster-up flag (Was=false) 2024-11-21T00:15:16,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,414 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T00:15:16,417 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:16,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:16,467 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T00:15:16,468 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:16,474 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T00:15:16,490 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(746): ClusterId : a60e3fdb-895f-4b66-963f-9689bc9d3dbc 2024-11-21T00:15:16,493 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T00:15:16,501 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T00:15:16,501 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T00:15:16,511 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T00:15:16,512 DEBUG [RS:0;5ed4808ef0e6:40705 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b29f114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5ed4808ef0e6/172.17.0.2:0 2024-11-21T00:15:16,532 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5ed4808ef0e6:40705 2024-11-21T00:15:16,536 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T00:15:16,536 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T00:15:16,536 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T00:15:16,539 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(2659): reportForDuty to master=5ed4808ef0e6,39691,1732148113019 with port=40705, startcode=1732148113864 2024-11-21T00:15:16,553 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T00:15:16,555 DEBUG [RS:0;5ed4808ef0e6:40705 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T00:15:16,566 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T00:15:16,575 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T00:15:16,582 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5ed4808ef0e6,39691,1732148113019 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T00:15:16,596 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:15:16,596 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:15:16,596 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:15:16,597 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:15:16,598 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5ed4808ef0e6:0, corePoolSize=10, maxPoolSize=10 2024-11-21T00:15:16,598 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,600 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T00:15:16,601 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,608 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732148146608 2024-11-21T00:15:16,610 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T00:15:16,612 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:15:16,613 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T00:15:16,618 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T00:15:16,619 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T00:15:16,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T00:15:16,619 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T00:15:16,619 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T00:15:16,620 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T00:15:16,625 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,629 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:16,630 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T00:15:16,633 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39691 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:16,635 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T00:15:16,637 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T00:15:16,637 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T00:15:16,640 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T00:15:16,640 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T00:15:16,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741831_1007 (size=1321) 2024-11-21T00:15:16,649 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148116642,5,FailOnTimeoutGroup] 2024-11-21T00:15:16,650 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148116650,5,FailOnTimeoutGroup] 2024-11-21T00:15:16,650 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,651 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T00:15:16,652 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,653 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,669 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T00:15:16,670 WARN [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T00:15:16,771 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(2659): reportForDuty to master=5ed4808ef0e6,39691,1732148113019 with port=40705, startcode=1732148113864 2024-11-21T00:15:16,774 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39691 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39691 {}] master.ServerManager(517): Registering regionserver=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,784 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:15:16,784 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36975 2024-11-21T00:15:16,784 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T00:15:16,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T00:15:16,804 DEBUG [RS:0;5ed4808ef0e6:40705 {}] zookeeper.ZKUtil(111): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,805 WARN [RS:0;5ed4808ef0e6:40705 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T00:15:16,805 INFO [RS:0;5ed4808ef0e6:40705 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:15:16,805 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,807 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5ed4808ef0e6,40705,1732148113864] 2024-11-21T00:15:16,833 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T00:15:16,850 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T00:15:16,855 INFO [RS:0;5ed4808ef0e6:40705 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T00:15:16,855 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,857 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T00:15:16,864 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T00:15:16,866 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,866 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,867 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,867 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,867 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,867 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,867 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5ed4808ef0e6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T00:15:16,868 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,868 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,868 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,868 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,868 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,869 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:15:16,869 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5ed4808ef0e6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T00:15:16,869 DEBUG [RS:0;5ed4808ef0e6:40705 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T00:15:16,870 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,870 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,870 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,870 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,870 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,871 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,40705,1732148113864-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T00:15:16,892 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T00:15:16,893 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,40705,1732148113864-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,894 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,894 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.Replication(171): 5ed4808ef0e6,40705,1732148113864 started 2024-11-21T00:15:16,915 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:16,915 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1482): Serving as 5ed4808ef0e6,40705,1732148113864, RpcServer on 5ed4808ef0e6/172.17.0.2:40705, sessionid=0x1015abea6c30001 2024-11-21T00:15:16,916 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T00:15:16,917 DEBUG [RS:0;5ed4808ef0e6:40705 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,917 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5ed4808ef0e6,40705,1732148113864' 2024-11-21T00:15:16,917 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T00:15:16,919 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T00:15:16,922 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T00:15:16,922 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T00:15:16,922 DEBUG [RS:0;5ed4808ef0e6:40705 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:16,922 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5ed4808ef0e6,40705,1732148113864' 2024-11-21T00:15:16,922 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T00:15:16,923 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T00:15:16,925 DEBUG [RS:0;5ed4808ef0e6:40705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T00:15:16,925 INFO [RS:0;5ed4808ef0e6:40705 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T00:15:16,925 INFO [RS:0;5ed4808ef0e6:40705 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T00:15:17,031 INFO [RS:0;5ed4808ef0e6:40705 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T00:15:17,036 INFO [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C40705%2C1732148113864, suffix=, logDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864, archiveDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/oldWALs, maxLogs=32 2024-11-21T00:15:17,051 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T00:15:17,052 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:15:17,055 DEBUG [RS:0;5ed4808ef0e6:40705 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864/5ed4808ef0e6%2C40705%2C1732148113864.1732148117039, exclude list is [], retry=0 2024-11-21T00:15:17,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45765,DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a,DISK] 2024-11-21T00:15:17,068 INFO [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864/5ed4808ef0e6%2C40705%2C1732148113864.1732148117039 2024-11-21T00:15:17,069 DEBUG [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38555:38555)] 2024-11-21T00:15:17,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741833_1009 (size=32) 2024-11-21T00:15:17,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:17,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T00:15:17,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T00:15:17,092 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T00:15:17,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T00:15:17,097 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T00:15:17,103 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T00:15:17,103 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T00:15:17,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T00:15:17,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T00:15:17,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740 2024-11-21T00:15:17,115 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740 2024-11-21T00:15:17,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T00:15:17,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T00:15:17,121 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:15:17,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T00:15:17,136 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:15:17,138 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71468500, jitterRate=0.06496363878250122}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:15:17,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732148117078Initializing all the Stores at 1732148117085 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117085Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117088 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148117088Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117088Cleaning up temporary data from old regions at 1732148117120 (+32 ms)Region opened successfully at 1732148117141 (+21 ms) 2024-11-21T00:15:17,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T00:15:17,141 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T00:15:17,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T00:15:17,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T00:15:17,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T00:15:17,147 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T00:15:17,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732148117141Disabling compacts and flushes for region at 1732148117141Disabling writes for close at 1732148117143 (+2 ms)Writing region close event to WAL at 1732148117146 (+3 ms)Closed at 1732148117147 (+1 ms) 2024-11-21T00:15:17,151 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:15:17,152 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T00:15:17,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T00:15:17,171 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T00:15:17,176 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T00:15:17,330 DEBUG [5ed4808ef0e6:39691 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T00:15:17,344 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:17,352 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5ed4808ef0e6,40705,1732148113864, state=OPENING 2024-11-21T00:15:17,372 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T00:15:17,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:17,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:15:17,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:15:17,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:15:17,386 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T00:15:17,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:17,565 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T00:15:17,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56573, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T00:15:17,582 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T00:15:17,582 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:15:17,583 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T00:15:17,589 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C40705%2C1732148113864.meta, suffix=.meta, logDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864, archiveDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/oldWALs, maxLogs=32 2024-11-21T00:15:17,612 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864/5ed4808ef0e6%2C40705%2C1732148113864.meta.1732148117592.meta, exclude list is [], retry=0 2024-11-21T00:15:17,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45765,DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a,DISK] 2024-11-21T00:15:17,624 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/WALs/5ed4808ef0e6,40705,1732148113864/5ed4808ef0e6%2C40705%2C1732148113864.meta.1732148117592.meta 2024-11-21T00:15:17,625 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38555:38555)] 2024-11-21T00:15:17,626 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:15:17,628 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T00:15:17,631 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T00:15:17,637 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T00:15:17,643 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T00:15:17,643 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:17,644 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T00:15:17,644 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T00:15:17,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T00:15:17,655 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T00:15:17,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T00:15:17,660 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T00:15:17,660 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T00:15:17,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T00:15:17,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T00:15:17,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T00:15:17,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:17,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:15:17,668 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T00:15:17,669 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740 2024-11-21T00:15:17,672 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740 2024-11-21T00:15:17,674 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T00:15:17,675 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T00:15:17,676 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:15:17,678 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T00:15:17,680 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59463486, jitterRate=-0.1139250099658966}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:15:17,680 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T00:15:17,681 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732148117645Writing region info on filesystem at 1732148117645Initializing all the Stores at 1732148117652 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117652Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117652Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148117652Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148117652Cleaning up temporary data from old regions at 1732148117675 (+23 ms)Running coprocessor post-open hooks at 1732148117680 (+5 ms)Region opened successfully at 1732148117681 (+1 ms) 2024-11-21T00:15:17,687 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732148117557 2024-11-21T00:15:17,697 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T00:15:17,698 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T00:15:17,699 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:17,702 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5ed4808ef0e6,40705,1732148113864, state=OPEN 2024-11-21T00:15:17,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T00:15:17,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T00:15:17,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:15:17,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:15:17,732 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:17,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T00:15:17,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,40705,1732148113864 in 344 msec 2024-11-21T00:15:17,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T00:15:17,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 584 msec 2024-11-21T00:15:17,750 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:15:17,750 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T00:15:17,769 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:17,770 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:17,788 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:17,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33771, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:17,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3060 sec 2024-11-21T00:15:17,811 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732148117810, completionTime=-1 2024-11-21T00:15:17,814 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T00:15:17,815 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-21T00:15:17,846 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-21T00:15:17,846 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732148177846 2024-11-21T00:15:17,846 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732148237846 2024-11-21T00:15:17,846 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-21T00:15:17,849 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,849 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,849 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,851 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5ed4808ef0e6:39691, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,851 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,851 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:17,857 DEBUG [master/5ed4808ef0e6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T00:15:17,900 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.821sec 2024-11-21T00:15:17,902 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T00:15:17,903 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T00:15:17,905 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T00:15:17,906 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T00:15:17,906 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T00:15:17,907 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T00:15:17,908 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T00:15:17,923 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T00:15:17,924 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T00:15:17,924 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,39691,1732148113019-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:15:18,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:18,003 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T00:15:18,003 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T00:15:18,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:18,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:18,017 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8c0cff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:18,028 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:18,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:18,032 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:18,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:18,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:18,074 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=820, ProcessCount=11, AvailableMemoryMB=1374 2024-11-21T00:15:18,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:15:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61e4ff20 2024-11-21T00:15:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:15:18,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:15:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:15:18,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:15:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:18,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:15:18,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T00:15:18,117 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:18,119 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:15:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:15:18,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741835_1011 (size=1061) 2024-11-21T00:15:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:15:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:15:18,537 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:15:18,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741836_1012 (size=53) 2024-11-21T00:15:18,553 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:18,553 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a1e0819c56248c71d5523ede3a8399ba, disabling compactions & flushes 2024-11-21T00:15:18,553 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. after waiting 0 ms 2024-11-21T00:15:18,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,554 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1e0819c56248c71d5523ede3a8399ba: Waiting for close lock at 1732148118553Disabling compacts and flushes for region at 1732148118553Disabling writes for close at 1732148118554 (+1 ms)Writing region close event to WAL at 1732148118554Closed at 1732148118554 2024-11-21T00:15:18,558 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:15:18,564 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148118558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148118558"}]},"ts":"1732148118558"} 2024-11-21T00:15:18,572 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:15:18,575 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:15:18,579 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148118575"}]},"ts":"1732148118575"} 2024-11-21T00:15:18,588 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:15:18,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, ASSIGN}] 2024-11-21T00:15:18,596 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, ASSIGN 2024-11-21T00:15:18,598 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:15:18,752 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1e0819c56248c71d5523ede3a8399ba, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:15:18,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, ASSIGN because future has completed 2024-11-21T00:15:18,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:18,936 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,937 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:15:18,937 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,937 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:18,937 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,938 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,939 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,943 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:18,944 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1e0819c56248c71d5523ede3a8399ba columnFamilyName A 2024-11-21T00:15:18,944 DEBUG [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:18,946 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(327): Store=a1e0819c56248c71d5523ede3a8399ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:18,946 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,949 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:18,949 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1e0819c56248c71d5523ede3a8399ba columnFamilyName B 2024-11-21T00:15:18,949 DEBUG [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:18,951 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(327): Store=a1e0819c56248c71d5523ede3a8399ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:18,951 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,954 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:18,954 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1e0819c56248c71d5523ede3a8399ba columnFamilyName C 2024-11-21T00:15:18,955 DEBUG [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:18,956 INFO [StoreOpener-a1e0819c56248c71d5523ede3a8399ba-1 {}] regionserver.HStore(327): Store=a1e0819c56248c71d5523ede3a8399ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:18,957 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,957 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,959 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,960 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,963 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,963 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,964 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:15:18,966 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,970 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:15:18,971 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a1e0819c56248c71d5523ede3a8399ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73422250, jitterRate=0.09407678246498108}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:15:18,972 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:18,973 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a1e0819c56248c71d5523ede3a8399ba: Running coprocessor pre-open hook at 1732148118938Writing region info on filesystem at 1732148118938Initializing all the Stores at 1732148118939 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148118939Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148118939Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148118939Cleaning up temporary data from old regions at 1732148118963 (+24 ms)Running coprocessor post-open hooks at 1732148118972 (+9 ms)Region opened successfully at 1732148118972 2024-11-21T00:15:18,975 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., pid=6, masterSystemTime=1732148118925 2024-11-21T00:15:18,979 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,979 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:18,980 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1e0819c56248c71d5523ede3a8399ba, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:18,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:15:18,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T00:15:18,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 in 220 msec 2024-11-21T00:15:19,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T00:15:19,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, ASSIGN in 404 msec 2024-11-21T00:15:19,005 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:15:19,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148119005"}]},"ts":"1732148119005"} 2024-11-21T00:15:19,011 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:15:19,014 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:15:19,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-21T00:15:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:15:19,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:19,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,280 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17282950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,283 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:15:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24720917 2024-11-21T00:15:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:15:19,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:15:19,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,311 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c7aae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,346 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fcd423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4787ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,354 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50131031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83804f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,364 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ba8772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,367 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272ed7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:19,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:19,375 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441ebf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:19,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b6ade8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:19,383 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,383 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,383 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,384 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,385 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,386 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,404 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T00:15:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:19,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:19,413 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,415 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,415 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:19,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:19,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,426 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:19,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:15:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:19,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:15:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:19,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:19,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:19,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148179663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148179663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148179666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148179671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148179676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/f12aa4db8b0e431d86cd3999a6d8d816 is 50, key is test_row_0/A:col10/1732148119485/Put/seqid=0 2024-11-21T00:15:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:19,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741837_1013 (size=12001) 2024-11-21T00:15:19,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148179790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148179790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148179798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148179798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148179811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:19,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:19,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148180008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148180009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148180017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148180018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148180029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:20,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/f12aa4db8b0e431d86cd3999a6d8d816 2024-11-21T00:15:20,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/61907d788f6c4e92b104c0e80fa02d54 is 50, key is test_row_0/B:col10/1732148119485/Put/seqid=0 2024-11-21T00:15:20,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741838_1014 (size=12001) 2024-11-21T00:15:20,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148180328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148180329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148180329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148180330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148180347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:20,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:20,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/61907d788f6c4e92b104c0e80fa02d54 2024-11-21T00:15:20,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/033a0d8ece22451fb241805c27303104 is 50, key is test_row_0/C:col10/1732148119485/Put/seqid=0 2024-11-21T00:15:20,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741839_1015 (size=12001) 2024-11-21T00:15:20,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/033a0d8ece22451fb241805c27303104 2024-11-21T00:15:20,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/f12aa4db8b0e431d86cd3999a6d8d816 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816 2024-11-21T00:15:20,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:15:20,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/61907d788f6c4e92b104c0e80fa02d54 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54 2024-11-21T00:15:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:15:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/033a0d8ece22451fb241805c27303104 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104 2024-11-21T00:15:20,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:15:20,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a1e0819c56248c71d5523ede3a8399ba in 1389ms, sequenceid=13, compaction requested=false 2024-11-21T00:15:20,888 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T00:15:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:20,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T00:15:20,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:20,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:20,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:20,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/9a8654bccb7e460e9fc1efaa1f4f6d18 is 50, key is test_row_0/A:col10/1732148120886/Put/seqid=0 2024-11-21T00:15:20,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741840_1016 (size=16681) 2024-11-21T00:15:20,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/9a8654bccb7e460e9fc1efaa1f4f6d18 2024-11-21T00:15:21,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/25a459e0a6dc464ca5242def6a3e872c is 50, key is test_row_0/B:col10/1732148120886/Put/seqid=0 2024-11-21T00:15:21,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741841_1017 (size=12001) 2024-11-21T00:15:21,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/25a459e0a6dc464ca5242def6a3e872c 2024-11-21T00:15:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148180987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148180994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148180999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148181019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148181025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ceb98e1512c84622b6763613553f139c is 50, key is test_row_0/C:col10/1732148120886/Put/seqid=0 2024-11-21T00:15:21,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:21,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:21,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:21,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741842_1018 (size=12001) 2024-11-21T00:15:21,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ceb98e1512c84622b6763613553f139c 2024-11-21T00:15:21,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/9a8654bccb7e460e9fc1efaa1f4f6d18 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18 2024-11-21T00:15:21,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18, entries=250, sequenceid=35, filesize=16.3 K 2024-11-21T00:15:21,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/25a459e0a6dc464ca5242def6a3e872c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c 2024-11-21T00:15:21,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148181158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148181168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148181169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148181169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148181171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c, entries=150, sequenceid=35, filesize=11.7 K 2024-11-21T00:15:21,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ceb98e1512c84622b6763613553f139c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c 2024-11-21T00:15:21,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c, entries=150, sequenceid=35, filesize=11.7 K 2024-11-21T00:15:21,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for a1e0819c56248c71d5523ede3a8399ba in 308ms, sequenceid=35, compaction requested=false 2024-11-21T00:15:21,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:21,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:15:21,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:21,256 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:15:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/fb24f169bb2a49db89ebd0b1d7ff12d9 is 50, key is test_row_0/A:col10/1732148120988/Put/seqid=0 2024-11-21T00:15:21,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741843_1019 (size=12001) 2024-11-21T00:15:21,342 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/fb24f169bb2a49db89ebd0b1d7ff12d9 2024-11-21T00:15:21,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/ec28a4540a844d6797a8347a8a0737f8 is 50, key is test_row_0/B:col10/1732148120988/Put/seqid=0 2024-11-21T00:15:21,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741844_1020 (size=12001) 2024-11-21T00:15:21,413 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/ec28a4540a844d6797a8347a8a0737f8 2024-11-21T00:15:21,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d45bd7f3db09476e8f6dc0282ed82398 is 50, key is test_row_0/C:col10/1732148120988/Put/seqid=0 2024-11-21T00:15:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741845_1021 (size=12001) 2024-11-21T00:15:21,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148121500, max timestamp=1732148121528Num uniques 859; 2024-11-21T00:15:21,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:21,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148121500, max timestamp=1732148121528Num uniques 859; 2024-11-21T00:15:21,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:21,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148121500, max timestamp=1732148121528Num uniques 859; 2024-11-21T00:15:21,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148181576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148181582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148181585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148181592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148181594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148181697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148181709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148181709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148181716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148181727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,860 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d45bd7f3db09476e8f6dc0282ed82398 2024-11-21T00:15:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/fb24f169bb2a49db89ebd0b1d7ff12d9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9 2024-11-21T00:15:21,900 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9, entries=150, sequenceid=46, filesize=11.7 K 2024-11-21T00:15:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/ec28a4540a844d6797a8347a8a0737f8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8 2024-11-21T00:15:21,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148181917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148181918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,937 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8, entries=150, sequenceid=46, filesize=11.7 K 2024-11-21T00:15:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148181928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d45bd7f3db09476e8f6dc0282ed82398 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398 2024-11-21T00:15:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148181937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148181947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:21,972 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398, entries=150, sequenceid=46, filesize=11.7 K 2024-11-21T00:15:21,977 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a1e0819c56248c71d5523ede3a8399ba in 720ms, sequenceid=46, compaction requested=true 2024-11-21T00:15:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T00:15:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T00:15:21,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T00:15:21,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5620 sec 2024-11-21T00:15:21,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5990 sec 2024-11-21T00:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:22,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:22,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/075e7d59528346d599c8d05eb46dd571 is 50, key is test_row_0/A:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:22,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148182270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148182270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741846_1022 (size=14341) 2024-11-21T00:15:22,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/075e7d59528346d599c8d05eb46dd571 2024-11-21T00:15:22,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148182280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148182288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148182294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/0e2a68cdd21c42289a10672cec0df00b is 50, key is test_row_0/B:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:22,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741847_1023 (size=12001) 2024-11-21T00:15:22,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/0e2a68cdd21c42289a10672cec0df00b 2024-11-21T00:15:22,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148182386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148182397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148182407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/40531c8858df404cb73bb7d1b1319c5d is 50, key is test_row_0/C:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148182416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148182416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741848_1024 (size=12001) 2024-11-21T00:15:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148182597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148182606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148182617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/40531c8858df404cb73bb7d1b1319c5d 2024-11-21T00:15:22,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/075e7d59528346d599c8d05eb46dd571 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571 2024-11-21T00:15:22,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571, entries=200, sequenceid=81, filesize=14.0 K 2024-11-21T00:15:22,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/0e2a68cdd21c42289a10672cec0df00b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b 2024-11-21T00:15:22,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T00:15:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148182907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148182921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/40531c8858df404cb73bb7d1b1319c5d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d 2024-11-21T00:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:22,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T00:15:22,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for a1e0819c56248c71d5523ede3a8399ba in 704ms, sequenceid=81, compaction requested=true 2024-11-21T00:15:22,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:22,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:22,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:22,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:22,956 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:22,960 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:22,962 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:22,963 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:22,963 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=46.9 K 2024-11-21T00:15:22,963 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:22,964 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:22,964 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:22,964 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=53.7 K 2024-11-21T00:15:22,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:22,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:22,965 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12aa4db8b0e431d86cd3999a6d8d816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148119471 2024-11-21T00:15:22,967 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8654bccb7e460e9fc1efaa1f4f6d18, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732148119661 2024-11-21T00:15:22,969 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 61907d788f6c4e92b104c0e80fa02d54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148119471 2024-11-21T00:15:22,969 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb24f169bb2a49db89ebd0b1d7ff12d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732148120988 2024-11-21T00:15:22,971 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a459e0a6dc464ca5242def6a3e872c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732148119661 2024-11-21T00:15:22,972 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ec28a4540a844d6797a8347a8a0737f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732148120988 2024-11-21T00:15:22,972 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2a68cdd21c42289a10672cec0df00b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121580 2024-11-21T00:15:22,973 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075e7d59528346d599c8d05eb46dd571, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121562 2024-11-21T00:15:22,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/0486aa891cb04b4da9646f13dc267b9f is 50, key is test_row_0/A:col10/1732148122262/Put/seqid=0 2024-11-21T00:15:23,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741849_1025 (size=16681) 2024-11-21T00:15:23,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/0486aa891cb04b4da9646f13dc267b9f 2024-11-21T00:15:23,050 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:23,052 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/6e7c4b30d53c4cd89512fba931596ee9 is 50, key is test_row_0/B:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:23,050 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#13 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:23,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/56ea6f5532014405b23e9b6e610a6218 is 50, key is test_row_0/B:col10/1732148122262/Put/seqid=0 2024-11-21T00:15:23,058 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/e88ce29c33b747fb96c52567b7bf1c23 is 50, key is test_row_0/A:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148183070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148183075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741851_1027 (size=12139) 2024-11-21T00:15:23,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148183087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741850_1026 (size=12139) 2024-11-21T00:15:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741852_1028 (size=12001) 2024-11-21T00:15:23,162 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:15:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148183217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148183227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148183227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,251 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T00:15:23,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T00:15:23,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T00:15:23,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T00:15:23,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T00:15:23,418 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T00:15:23,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T00:15:23,419 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T00:15:23,420 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T00:15:23,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148183427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148183437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148183437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148183437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148183446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,519 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/e88ce29c33b747fb96c52567b7bf1c23 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/e88ce29c33b747fb96c52567b7bf1c23 2024-11-21T00:15:23,527 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/6e7c4b30d53c4cd89512fba931596ee9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/6e7c4b30d53c4cd89512fba931596ee9 2024-11-21T00:15:23,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/56ea6f5532014405b23e9b6e610a6218 2024-11-21T00:15:23,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/fa4dcb517ad9481ea9beb0c6c6cba9ec is 50, key is test_row_0/C:col10/1732148122262/Put/seqid=0 2024-11-21T00:15:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:15:23,568 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:23,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T00:15:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:23,572 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:23,573 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into 6e7c4b30d53c4cd89512fba931596ee9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:23,573 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:23,574 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:23,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:23,576 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into e88ce29c33b747fb96c52567b7bf1c23(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:23,576 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:23,577 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=12, startTime=1732148122943; duration=0sec 2024-11-21T00:15:23,577 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=12, startTime=1732148122953; duration=0sec 2024-11-21T00:15:23,577 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:23,578 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:23,578 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:23,583 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:23,583 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:23,583 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:23,584 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=46.9 K 2024-11-21T00:15:23,585 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 033a0d8ece22451fb241805c27303104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148119471 2024-11-21T00:15:23,586 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb98e1512c84622b6763613553f139c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732148119661 2024-11-21T00:15:23,587 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:23,587 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:23,587 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d45bd7f3db09476e8f6dc0282ed82398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732148120988 2024-11-21T00:15:23,588 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40531c8858df404cb73bb7d1b1319c5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121580 2024-11-21T00:15:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741853_1029 (size=12001) 2024-11-21T00:15:23,634 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#17 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:23,636 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/75e8492c32a44d09ad2b50ef4b287bc4 is 50, key is test_row_0/C:col10/1732148121586/Put/seqid=0 2024-11-21T00:15:23,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741854_1030 (size=12139) 2024-11-21T00:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:23,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:15:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:23,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148183736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148183757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148183757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:23,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:15:23,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:23,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:23,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:23,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:24,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/fa4dcb517ad9481ea9beb0c6c6cba9ec 2024-11-21T00:15:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/0486aa891cb04b4da9646f13dc267b9f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f 2024-11-21T00:15:24,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:15:24,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:24,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:24,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:24,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:24,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:24,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f, entries=250, sequenceid=96, filesize=16.3 K 2024-11-21T00:15:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/56ea6f5532014405b23e9b6e610a6218 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218 2024-11-21T00:15:24,070 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/75e8492c32a44d09ad2b50ef4b287bc4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/75e8492c32a44d09ad2b50ef4b287bc4 2024-11-21T00:15:24,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218, entries=150, sequenceid=96, filesize=11.7 K 2024-11-21T00:15:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/fa4dcb517ad9481ea9beb0c6c6cba9ec as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec 2024-11-21T00:15:24,092 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into 75e8492c32a44d09ad2b50ef4b287bc4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:24,092 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:24,108 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=12, startTime=1732148122953; duration=0sec 2024-11-21T00:15:24,108 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:24,109 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:24,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec, entries=150, sequenceid=96, filesize=11.7 K 2024-11-21T00:15:24,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a1e0819c56248c71d5523ede3a8399ba in 1148ms, sequenceid=96, compaction requested=false 2024-11-21T00:15:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:24,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:15:24,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:24,200 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/976deb4a40f1477594892c6c4d0f61a3 is 50, key is test_row_0/A:col10/1732148123083/Put/seqid=0 2024-11-21T00:15:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741855_1031 (size=12001) 2024-11-21T00:15:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:24,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148184315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148184322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148184330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148184456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148184456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148184456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,461 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148184457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148184457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,466 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,645 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/976deb4a40f1477594892c6c4d0f61a3 2024-11-21T00:15:24,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148184668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148184676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148184677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1a05a38de7ea4a3f971a75a33a1b276c is 50, key is test_row_0/B:col10/1732148123083/Put/seqid=0 2024-11-21T00:15:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:24,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741856_1032 (size=12001) 2024-11-21T00:15:24,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1a05a38de7ea4a3f971a75a33a1b276c 2024-11-21T00:15:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/5d2e274de3f045b5905b8f74ff2247f8 is 50, key is test_row_0/C:col10/1732148123083/Put/seqid=0 2024-11-21T00:15:24,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741857_1033 (size=12001) 2024-11-21T00:15:24,773 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/5d2e274de3f045b5905b8f74ff2247f8 2024-11-21T00:15:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/976deb4a40f1477594892c6c4d0f61a3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3 2024-11-21T00:15:24,806 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:15:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1a05a38de7ea4a3f971a75a33a1b276c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c 2024-11-21T00:15:24,836 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:15:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/5d2e274de3f045b5905b8f74ff2247f8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8 2024-11-21T00:15:24,851 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:15:24,856 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a1e0819c56248c71d5523ede3a8399ba in 656ms, sequenceid=116, compaction requested=true 2024-11-21T00:15:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T00:15:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T00:15:24,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T00:15:24,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2870 sec 2024-11-21T00:15:24,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2990 sec 2024-11-21T00:15:24,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:25,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/edaa80ec441b43ae8fb5d9bb32b2eedb is 50, key is test_row_0/A:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741858_1034 (size=9707) 2024-11-21T00:15:25,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/edaa80ec441b43ae8fb5d9bb32b2eedb 2024-11-21T00:15:25,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148185050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/f95420dc119245e2b7f20c25fed6787c is 50, key is test_row_0/B:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148185063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148185064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741859_1035 (size=9707) 2024-11-21T00:15:25,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/f95420dc119245e2b7f20c25fed6787c 2024-11-21T00:15:25,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/481105742c8d48ddae719a9471f6e8b0 is 50, key is test_row_0/C:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741860_1036 (size=9707) 2024-11-21T00:15:25,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/481105742c8d48ddae719a9471f6e8b0 2024-11-21T00:15:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/edaa80ec441b43ae8fb5d9bb32b2eedb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb 2024-11-21T00:15:25,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb, entries=100, sequenceid=132, filesize=9.5 K 2024-11-21T00:15:25,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/f95420dc119245e2b7f20c25fed6787c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c 2024-11-21T00:15:25,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c, entries=100, sequenceid=132, filesize=9.5 K 2024-11-21T00:15:25,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/481105742c8d48ddae719a9471f6e8b0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0 2024-11-21T00:15:25,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0, entries=100, sequenceid=132, filesize=9.5 K 2024-11-21T00:15:25,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for a1e0819c56248c71d5523ede3a8399ba in 166ms, sequenceid=132, compaction requested=true 2024-11-21T00:15:25,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:25,156 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:25,156 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:25,159 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:25,160 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:25,160 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:25,160 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:25,160 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:25,160 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:25,160 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/6e7c4b30d53c4cd89512fba931596ee9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=44.8 K 2024-11-21T00:15:25,160 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/e88ce29c33b747fb96c52567b7bf1c23, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=49.3 K 2024-11-21T00:15:25,161 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e7c4b30d53c4cd89512fba931596ee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121580 2024-11-21T00:15:25,162 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ea6f5532014405b23e9b6e610a6218, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148122262 2024-11-21T00:15:25,162 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a05a38de7ea4a3f971a75a33a1b276c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148123047 2024-11-21T00:15:25,162 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting e88ce29c33b747fb96c52567b7bf1c23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121580 2024-11-21T00:15:25,163 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f95420dc119245e2b7f20c25fed6787c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148124288 2024-11-21T00:15:25,163 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0486aa891cb04b4da9646f13dc267b9f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148122262 2024-11-21T00:15:25,164 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 976deb4a40f1477594892c6c4d0f61a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148123047 2024-11-21T00:15:25,164 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting edaa80ec441b43ae8fb5d9bb32b2eedb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148124288 2024-11-21T00:15:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:15:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:25,210 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#24 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:25,211 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a126ddf27fb54fd699e2dd10b82d756e is 50, key is test_row_0/A:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,220 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:25,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148185212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148185209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,221 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/73d460b96aa644d58d62cc5648360603 is 50, key is test_row_0/B:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148185217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7d5aa08459f143e6bcea11e7bb22b251 is 50, key is test_row_0/A:col10/1732148125169/Put/seqid=0 2024-11-21T00:15:25,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741862_1038 (size=12325) 2024-11-21T00:15:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741861_1037 (size=12325) 2024-11-21T00:15:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741863_1039 (size=12151) 2024-11-21T00:15:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148185327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148185327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148185327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148185537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148185537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148185549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7d5aa08459f143e6bcea11e7bb22b251 2024-11-21T00:15:25,702 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/73d460b96aa644d58d62cc5648360603 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/73d460b96aa644d58d62cc5648360603 2024-11-21T00:15:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:15:25,717 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:25,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T00:15:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:15:25,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:25,722 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a126ddf27fb54fd699e2dd10b82d756e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a126ddf27fb54fd699e2dd10b82d756e 2024-11-21T00:15:25,723 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:25,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:25,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/d8cb5e6b646040b194201d8e34ace999 is 50, key is test_row_0/B:col10/1732148125169/Put/seqid=0 2024-11-21T00:15:25,731 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into 73d460b96aa644d58d62cc5648360603(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:25,731 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:25,731 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=12, startTime=1732148125156; duration=0sec 2024-11-21T00:15:25,732 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:25,732 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:25,732 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:25,736 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:25,736 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:25,736 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:25,736 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/75e8492c32a44d09ad2b50ef4b287bc4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=44.8 K 2024-11-21T00:15:25,738 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into a126ddf27fb54fd699e2dd10b82d756e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:25,738 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:25,738 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=12, startTime=1732148125156; duration=0sec 2024-11-21T00:15:25,739 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:25,739 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:25,740 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e8492c32a44d09ad2b50ef4b287bc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732148121580 2024-11-21T00:15:25,742 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4dcb517ad9481ea9beb0c6c6cba9ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148122262 2024-11-21T00:15:25,744 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d2e274de3f045b5905b8f74ff2247f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148123047 2024-11-21T00:15:25,744 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 481105742c8d48ddae719a9471f6e8b0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148124288 2024-11-21T00:15:25,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741864_1040 (size=12151) 2024-11-21T00:15:25,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/d8cb5e6b646040b194201d8e34ace999 2024-11-21T00:15:25,783 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:25,784 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/31c4621140d6455d8059167297cb8106 is 50, key is test_row_0/C:col10/1732148124288/Put/seqid=0 2024-11-21T00:15:25,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba72bd09f1984506a44d6107fc86327c is 50, key is test_row_0/C:col10/1732148125169/Put/seqid=0 2024-11-21T00:15:25,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741865_1041 (size=12325) 2024-11-21T00:15:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:15:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741866_1042 (size=12151) 2024-11-21T00:15:25,836 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/31c4621140d6455d8059167297cb8106 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/31c4621140d6455d8059167297cb8106 2024-11-21T00:15:25,848 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into 31c4621140d6455d8059167297cb8106(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:25,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:25,848 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=12, startTime=1732148125156; duration=0sec 2024-11-21T00:15:25,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148185847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148185856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148185867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:25,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:25,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:15:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:25,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:15:26,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:15:26,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:26,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba72bd09f1984506a44d6107fc86327c 2024-11-21T00:15:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7d5aa08459f143e6bcea11e7bb22b251 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251 2024-11-21T00:15:26,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:15:26,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/d8cb5e6b646040b194201d8e34ace999 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999 2024-11-21T00:15:26,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:15:26,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba72bd09f1984506a44d6107fc86327c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c 2024-11-21T00:15:26,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:15:26,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a1e0819c56248c71d5523ede3a8399ba in 1110ms, sequenceid=150, compaction requested=false 2024-11-21T00:15:26,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:15:26,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:15:26,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/10a73f2ab7ce478099e3d1410a71402d is 50, key is test_row_0/A:col10/1732148125205/Put/seqid=0 2024-11-21T00:15:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741867_1043 (size=12151) 2024-11-21T00:15:26,392 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/10a73f2ab7ce478099e3d1410a71402d 2024-11-21T00:15:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b512c5e1663546eeb2f1eaf3948ac278 is 50, key is test_row_0/B:col10/1732148125205/Put/seqid=0 2024-11-21T00:15:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148186432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148186435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148186438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148186467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,471 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:26,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741868_1044 (size=12151) 2024-11-21T00:15:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,476 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b512c5e1663546eeb2f1eaf3948ac278 2024-11-21T00:15:26,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148186477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,483 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba9b24e774f04e3f82ef700f7c83b6f7 is 50, key is test_row_0/C:col10/1732148125205/Put/seqid=0 2024-11-21T00:15:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148186547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148186547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148186547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741869_1045 (size=12151) 2024-11-21T00:15:26,560 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba9b24e774f04e3f82ef700f7c83b6f7 2024-11-21T00:15:26,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/10a73f2ab7ce478099e3d1410a71402d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d 2024-11-21T00:15:26,592 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T00:15:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b512c5e1663546eeb2f1eaf3948ac278 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278 2024-11-21T00:15:26,606 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T00:15:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/ba9b24e774f04e3f82ef700f7c83b6f7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7 2024-11-21T00:15:26,634 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T00:15:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a1e0819c56248c71d5523ede3a8399ba in 290ms, sequenceid=167, compaction requested=true 2024-11-21T00:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T00:15:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T00:15:26,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T00:15:26,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 917 msec 2024-11-21T00:15:26,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 930 msec 2024-11-21T00:15:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:26,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:15:26,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:26,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4422d5498ec149bbbc7a70f64668befd is 50, key is test_row_0/A:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:26,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741870_1046 (size=14541) 2024-11-21T00:15:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148186793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4422d5498ec149bbbc7a70f64668befd 2024-11-21T00:15:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148186794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148186805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1e0e7a1265bc41389caa14ccdb27073b is 50, key is test_row_0/B:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:15:26,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:26,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T00:15:26,863 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:26,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:26,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:26,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741871_1047 (size=12151) 2024-11-21T00:15:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148186920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148186920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148186924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:27,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148187126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148187138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148187140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:27,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1e0e7a1265bc41389caa14ccdb27073b 2024-11-21T00:15:27,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6a2b824ac2ea40379f935499e2e41f07 is 50, key is test_row_0/C:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:27,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741872_1048 (size=12151) 2024-11-21T00:15:27,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6a2b824ac2ea40379f935499e2e41f07 2024-11-21T00:15:27,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4422d5498ec149bbbc7a70f64668befd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd 2024-11-21T00:15:27,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd, entries=200, sequenceid=187, filesize=14.2 K 2024-11-21T00:15:27,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1e0e7a1265bc41389caa14ccdb27073b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b 2024-11-21T00:15:27,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T00:15:27,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6a2b824ac2ea40379f935499e2e41f07 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07 2024-11-21T00:15:27,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T00:15:27,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1e0819c56248c71d5523ede3a8399ba in 619ms, sequenceid=187, compaction requested=true 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:27,384 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:27,384 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:27,387 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:27,387 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:27,387 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,387 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/73d460b96aa644d58d62cc5648360603, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=47.6 K 2024-11-21T00:15:27,388 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d460b96aa644d58d62cc5648360603, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148123066 2024-11-21T00:15:27,388 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51168 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:27,389 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:27,389 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,389 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a126ddf27fb54fd699e2dd10b82d756e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=50.0 K 2024-11-21T00:15:27,389 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d8cb5e6b646040b194201d8e34ace999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148125040 2024-11-21T00:15:27,389 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a126ddf27fb54fd699e2dd10b82d756e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148123066 2024-11-21T00:15:27,390 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b512c5e1663546eeb2f1eaf3948ac278, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732148125201 2024-11-21T00:15:27,390 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5aa08459f143e6bcea11e7bb22b251, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148125040 2024-11-21T00:15:27,390 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0e7a1265bc41389caa14ccdb27073b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:27,391 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a73f2ab7ce478099e3d1410a71402d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732148125201 2024-11-21T00:15:27,392 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4422d5498ec149bbbc7a70f64668befd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:27,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:27,447 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:27,465 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/bd001632ceb54ce8b4d17bc4729a3071 is 50, key is test_row_0/B:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:27,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/01888cf10ca9488ea98fe7a889c8ef17 is 50, key is test_row_0/A:col10/1732148126800/Put/seqid=0 2024-11-21T00:15:27,469 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#37 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:27,470 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/8131a9bf5d9c4c009205db855d926194 is 50, key is test_row_0/A:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:27,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741873_1049 (size=12561) 2024-11-21T00:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:27,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741875_1051 (size=14541) 2024-11-21T00:15:27,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/01888cf10ca9488ea98fe7a889c8ef17 2024-11-21T00:15:27,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148187524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148187539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741874_1050 (size=12561) 2024-11-21T00:15:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148187545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,552 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/bd001632ceb54ce8b4d17bc4729a3071 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/bd001632ceb54ce8b4d17bc4729a3071 2024-11-21T00:15:27,565 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/8131a9bf5d9c4c009205db855d926194 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/8131a9bf5d9c4c009205db855d926194 2024-11-21T00:15:27,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c6682fecfef5441f8a65732415a267a0 is 50, key is test_row_0/B:col10/1732148126800/Put/seqid=0 2024-11-21T00:15:27,571 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into bd001632ceb54ce8b4d17bc4729a3071(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:27,571 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:27,571 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=12, startTime=1732148127380; duration=0sec 2024-11-21T00:15:27,572 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:27,572 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:27,572 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:27,574 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:27,574 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:27,575 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,576 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/31c4621140d6455d8059167297cb8106, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=47.6 K 2024-11-21T00:15:27,581 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c4621140d6455d8059167297cb8106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148123066 2024-11-21T00:15:27,584 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into 8131a9bf5d9c4c009205db855d926194(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:27,584 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:27,584 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=12, startTime=1732148127380; duration=0sec 2024-11-21T00:15:27,584 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:27,584 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:27,586 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ba72bd09f1984506a44d6107fc86327c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148125040 2024-11-21T00:15:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741876_1052 (size=12151) 2024-11-21T00:15:27,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9b24e774f04e3f82ef700f7c83b6f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732148125201 2024-11-21T00:15:27,604 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2b824ac2ea40379f935499e2e41f07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:27,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c6682fecfef5441f8a65732415a267a0 2024-11-21T00:15:27,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/4ac0b4bb65c84a69b96b09db7e4adebc is 50, key is test_row_0/C:col10/1732148126800/Put/seqid=0 2024-11-21T00:15:27,650 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#41 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:27,651 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/1ef1eae5a9f940e4b776c59598e00d68 is 50, key is test_row_0/C:col10/1732148126435/Put/seqid=0 2024-11-21T00:15:27,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:27,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741877_1053 (size=12151) 2024-11-21T00:15:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148187657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/4ac0b4bb65c84a69b96b09db7e4adebc 2024-11-21T00:15:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148187657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148187660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741878_1054 (size=12561) 2024-11-21T00:15:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/01888cf10ca9488ea98fe7a889c8ef17 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17 2024-11-21T00:15:27,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17, entries=200, sequenceid=200, filesize=14.2 K 2024-11-21T00:15:27,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c6682fecfef5441f8a65732415a267a0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0 2024-11-21T00:15:27,706 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/1ef1eae5a9f940e4b776c59598e00d68 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/1ef1eae5a9f940e4b776c59598e00d68 2024-11-21T00:15:27,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T00:15:27,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/4ac0b4bb65c84a69b96b09db7e4adebc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc 2024-11-21T00:15:27,722 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into 1ef1eae5a9f940e4b776c59598e00d68(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:27,722 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:27,722 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=12, startTime=1732148127380; duration=0sec 2024-11-21T00:15:27,722 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:27,722 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:27,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T00:15:27,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a1e0819c56248c71d5523ede3a8399ba in 296ms, sequenceid=200, compaction requested=false 2024-11-21T00:15:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:27,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:15:27,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:27,826 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:15:27,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:27,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:27,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/64f1b87394fc4faa9dbba1bbc74b8690 is 50, key is test_row_0/A:col10/1732148127488/Put/seqid=0 2024-11-21T00:15:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:27,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741879_1055 (size=12151) 2024-11-21T00:15:27,900 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/64f1b87394fc4faa9dbba1bbc74b8690 2024-11-21T00:15:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148187907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148187910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148187911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:27,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58fc345bf35a49a9bc7ba898fd54ded4 is 50, key is test_row_0/B:col10/1732148127488/Put/seqid=0 2024-11-21T00:15:27,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741880_1056 (size=12151) 2024-11-21T00:15:27,942 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58fc345bf35a49a9bc7ba898fd54ded4 2024-11-21T00:15:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3d9d15ea8eaa46e3b5f9b04124560324 is 50, key is test_row_0/C:col10/1732148127488/Put/seqid=0 2024-11-21T00:15:27,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741881_1057 (size=12151) 2024-11-21T00:15:28,005 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3d9d15ea8eaa46e3b5f9b04124560324 2024-11-21T00:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148188017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/64f1b87394fc4faa9dbba1bbc74b8690 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690 2024-11-21T00:15:28,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148188027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148188033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,058 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T00:15:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58fc345bf35a49a9bc7ba898fd54ded4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4 2024-11-21T00:15:28,073 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T00:15:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3d9d15ea8eaa46e3b5f9b04124560324 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324 2024-11-21T00:15:28,084 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T00:15:28,094 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a1e0819c56248c71d5523ede3a8399ba in 267ms, sequenceid=222, compaction requested=true 2024-11-21T00:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T00:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T00:15:28,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T00:15:28,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-11-21T00:15:28,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2570 sec 2024-11-21T00:15:28,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:15:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:28,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a13a7ea10d7d4fad9277f934b125687c is 50, key is test_row_0/A:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741882_1058 (size=14541) 2024-11-21T00:15:28,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a13a7ea10d7d4fad9277f934b125687c 2024-11-21T00:15:28,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3ffa36fe20124540bcf8a2a891b8a5cd is 50, key is test_row_0/B:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:28,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148188311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148188315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148188318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741883_1059 (size=12151) 2024-11-21T00:15:28,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3ffa36fe20124540bcf8a2a891b8a5cd 2024-11-21T00:15:28,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d9e248c0fa2745d2a61bab6103d76e4d is 50, key is test_row_0/C:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148188430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741884_1060 (size=12151) 2024-11-21T00:15:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148188440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148188444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148188647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148188668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148188672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:28,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d9e248c0fa2745d2a61bab6103d76e4d 2024-11-21T00:15:28,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a13a7ea10d7d4fad9277f934b125687c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c 2024-11-21T00:15:28,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c, entries=200, sequenceid=236, filesize=14.2 K 2024-11-21T00:15:28,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3ffa36fe20124540bcf8a2a891b8a5cd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd 2024-11-21T00:15:28,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd, entries=150, sequenceid=236, filesize=11.9 K 2024-11-21T00:15:28,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d9e248c0fa2745d2a61bab6103d76e4d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d 2024-11-21T00:15:28,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-21T00:15:28,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a1e0819c56248c71d5523ede3a8399ba in 666ms, sequenceid=236, compaction requested=true 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:28,901 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:28,901 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:28,903 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:28,903 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:28,903 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:28,903 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:28,903 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/bd001632ceb54ce8b4d17bc4729a3071, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=47.9 K 2024-11-21T00:15:28,904 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:28,904 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:28,904 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/8131a9bf5d9c4c009205db855d926194, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=52.5 K 2024-11-21T00:15:28,904 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting bd001632ceb54ce8b4d17bc4729a3071, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:28,905 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8131a9bf5d9c4c009205db855d926194, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:28,905 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c6682fecfef5441f8a65732415a267a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732148126790 2024-11-21T00:15:28,906 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01888cf10ca9488ea98fe7a889c8ef17, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732148126782 2024-11-21T00:15:28,907 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 58fc345bf35a49a9bc7ba898fd54ded4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732148127488 2024-11-21T00:15:28,907 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ffa36fe20124540bcf8a2a891b8a5cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:28,908 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f1b87394fc4faa9dbba1bbc74b8690, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732148127488 2024-11-21T00:15:28,909 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13a7ea10d7d4fad9277f934b125687c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:28,935 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:28,936 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:28,936 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/a2ae3315bbf74c70a0fd4c35a7ebff29 is 50, key is test_row_0/B:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:28,936 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/d6db5eda52b94b0ba78fe16d8ccc87cd is 50, key is test_row_0/A:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:28,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:15:28,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:28,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:28,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:28,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:28,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741885_1061 (size=12697) 2024-11-21T00:15:29,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148189001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:15:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148189013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148189013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:29,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/bbc5723a5f294026b96a8a685992eb42 is 50, key is test_row_0/A:col10/1732148128294/Put/seqid=0 2024-11-21T00:15:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T00:15:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:29,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:29,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:29,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741886_1062 (size=12697) 2024-11-21T00:15:29,038 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/a2ae3315bbf74c70a0fd4c35a7ebff29 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a2ae3315bbf74c70a0fd4c35a7ebff29 2024-11-21T00:15:29,053 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into a2ae3315bbf74c70a0fd4c35a7ebff29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:29,053 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:29,053 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=12, startTime=1732148128901; duration=0sec 2024-11-21T00:15:29,053 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:29,054 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:29,054 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741887_1063 (size=12151) 2024-11-21T00:15:29,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/bbc5723a5f294026b96a8a685992eb42 2024-11-21T00:15:29,057 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:29,058 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:29,058 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,058 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/1ef1eae5a9f940e4b776c59598e00d68, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=47.9 K 2024-11-21T00:15:29,058 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/d6db5eda52b94b0ba78fe16d8ccc87cd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/d6db5eda52b94b0ba78fe16d8ccc87cd 2024-11-21T00:15:29,059 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef1eae5a9f940e4b776c59598e00d68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732148126413 2024-11-21T00:15:29,061 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac0b4bb65c84a69b96b09db7e4adebc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732148126790 2024-11-21T00:15:29,063 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9d15ea8eaa46e3b5f9b04124560324, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732148127488 2024-11-21T00:15:29,067 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e248c0fa2745d2a61bab6103d76e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:29,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/a1ea3ca117ce4550bb8f5634a33e70e2 is 50, key is test_row_0/B:col10/1732148128294/Put/seqid=0 2024-11-21T00:15:29,071 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into d6db5eda52b94b0ba78fe16d8ccc87cd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:29,072 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:29,073 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=12, startTime=1732148128901; duration=0sec 2024-11-21T00:15:29,073 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:29,073 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:29,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741888_1064 (size=12151) 2024-11-21T00:15:29,085 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:29,087 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/4af2fe501eb441f697d67fd1fe58a519 is 50, key is test_row_0/C:col10/1732148127905/Put/seqid=0 2024-11-21T00:15:29,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741889_1065 (size=12697) 2024-11-21T00:15:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148189127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,128 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/4af2fe501eb441f697d67fd1fe58a519 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4af2fe501eb441f697d67fd1fe58a519 2024-11-21T00:15:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148189133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148189133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,149 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into 4af2fe501eb441f697d67fd1fe58a519(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:29,149 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:29,149 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=12, startTime=1732148128901; duration=0sec 2024-11-21T00:15:29,149 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:29,149 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:29,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:29,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148189339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148189346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148189350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/a1ea3ca117ce4550bb8f5634a33e70e2 2024-11-21T00:15:29,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6e39be9caf4b425db27f3807954823c1 is 50, key is test_row_0/C:col10/1732148128294/Put/seqid=0 2024-11-21T00:15:29,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741890_1066 (size=12151) 2024-11-21T00:15:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148189663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148189664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148189658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:29,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6e39be9caf4b425db27f3807954823c1 2024-11-21T00:15:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/bbc5723a5f294026b96a8a685992eb42 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42 2024-11-21T00:15:29,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T00:15:29,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:29,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/a1ea3ca117ce4550bb8f5634a33e70e2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2 2024-11-21T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:29,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:30,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T00:15:30,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6e39be9caf4b425db27f3807954823c1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1 2024-11-21T00:15:30,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T00:15:30,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a1e0819c56248c71d5523ede3a8399ba in 1067ms, sequenceid=255, compaction requested=false 2024-11-21T00:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:30,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:30,155 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:30,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/76843f0dfb724a6bbd95a9b4d3c0b43c is 50, key is test_row_0/A:col10/1732148129008/Put/seqid=0 2024-11-21T00:15:30,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741891_1067 (size=12301) 2024-11-21T00:15:30,167 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/76843f0dfb724a6bbd95a9b4d3c0b43c 2024-11-21T00:15:30,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3141f1b528e24e2eb6b3c0ce16a37064 is 50, key is test_row_0/B:col10/1732148129008/Put/seqid=0 2024-11-21T00:15:30,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741892_1068 (size=12301) 2024-11-21T00:15:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:30,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148190221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148190224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148190225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148190337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148190337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148190337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49192 deadline: 1732148190496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49190 deadline: 1732148190516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,518 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in a1e0819c56248c71d5523ede3a8399ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148190546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148190546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148190547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,582 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3141f1b528e24e2eb6b3c0ce16a37064 2024-11-21T00:15:30,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/efcd1788931f431286b125d7c2c51b29 is 50, key is test_row_0/C:col10/1732148129008/Put/seqid=0 2024-11-21T00:15:30,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741893_1069 (size=12301) 2024-11-21T00:15:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148190857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148190857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148190867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:30,995 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/efcd1788931f431286b125d7c2c51b29 2024-11-21T00:15:31,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/76843f0dfb724a6bbd95a9b4d3c0b43c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c 2024-11-21T00:15:31,015 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:15:31,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3141f1b528e24e2eb6b3c0ce16a37064 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064 2024-11-21T00:15:31,025 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:15:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/efcd1788931f431286b125d7c2c51b29 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29 2024-11-21T00:15:31,034 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:15:31,036 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a1e0819c56248c71d5523ede3a8399ba in 880ms, sequenceid=271, compaction requested=true 2024-11-21T00:15:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T00:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T00:15:31,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T00:15:31,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-21T00:15:31,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0280 sec 2024-11-21T00:15:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:15:31,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:31,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T00:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:31,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:31,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:31,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:31,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T00:15:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:31,332 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:31,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4b15a19cbb2845e5ac33cf10cde61e00 is 50, key is test_row_0/A:col10/1732148130222/Put/seqid=0 2024-11-21T00:15:31,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741894_1070 (size=12301) 2024-11-21T00:15:31,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148191414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148191418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148191420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148191527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148191527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148191537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148191737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148191737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148191747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:31,816 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4b15a19cbb2845e5ac33cf10cde61e00 2024-11-21T00:15:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c73863e1944f4ae6b7254fd968083813 is 50, key is test_row_0/B:col10/1732148130222/Put/seqid=0 2024-11-21T00:15:31,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741895_1071 (size=12301) 2024-11-21T00:15:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148192047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148192047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148192069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,316 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c73863e1944f4ae6b7254fd968083813 2024-11-21T00:15:32,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/be59834300b34a7abb26df51498ecc73 is 50, key is test_row_0/C:col10/1732148130222/Put/seqid=0 2024-11-21T00:15:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:32,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741896_1072 (size=12301) 2024-11-21T00:15:32,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148192557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148192566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148192588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:32,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/be59834300b34a7abb26df51498ecc73 2024-11-21T00:15:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/4b15a19cbb2845e5ac33cf10cde61e00 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00 2024-11-21T00:15:32,796 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:15:32,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/c73863e1944f4ae6b7254fd968083813 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813 2024-11-21T00:15:32,814 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:15:32,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/be59834300b34a7abb26df51498ecc73 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73 2024-11-21T00:15:32,857 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:15:32,858 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for a1e0819c56248c71d5523ede3a8399ba in 1527ms, sequenceid=290, compaction requested=true 2024-11-21T00:15:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T00:15:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T00:15:32,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T00:15:32,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-21T00:15:32,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7030 sec 2024-11-21T00:15:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:15:33,357 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:33,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T00:15:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:15:33,368 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:33,369 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:33,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:15:33,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T00:15:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:33,525 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:33,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a203a2dea47b445fa9dc4955ad6f63c3 is 50, key is test_row_0/A:col10/1732148131413/Put/seqid=0 2024-11-21T00:15:33,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741897_1073 (size=12301) 2024-11-21T00:15:33,556 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a203a2dea47b445fa9dc4955ad6f63c3 2024-11-21T00:15:33,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/839601a3551f4f939d05dc952d1728fc is 50, key is test_row_0/B:col10/1732148131413/Put/seqid=0 2024-11-21T00:15:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:33,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:33,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741898_1074 (size=12301) 2024-11-21T00:15:33,599 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/839601a3551f4f939d05dc952d1728fc 2024-11-21T00:15:33,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3f3a7ccbfab2440ea6091136fad02d79 is 50, key is test_row_0/C:col10/1732148131413/Put/seqid=0 2024-11-21T00:15:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148193633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148193636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148193637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741899_1075 (size=12301) 2024-11-21T00:15:33,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3f3a7ccbfab2440ea6091136fad02d79 2024-11-21T00:15:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/a203a2dea47b445fa9dc4955ad6f63c3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3 2024-11-21T00:15:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:15:33,702 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T00:15:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/839601a3551f4f939d05dc952d1728fc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc 2024-11-21T00:15:33,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:33,765 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T00:15:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/3f3a7ccbfab2440ea6091136fad02d79 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79 2024-11-21T00:15:33,796 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T00:15:33,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a1e0819c56248c71d5523ede3a8399ba in 273ms, sequenceid=304, compaction requested=true 2024-11-21T00:15:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T00:15:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T00:15:33,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T00:15:33,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 436 msec 2024-11-21T00:15:33,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 455 msec 2024-11-21T00:15:33,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148133630, max timestamp=1732148133976Num uniques 859; 2024-11-21T00:15:33,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:33,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148133630, max timestamp=1732148133976Num uniques 859; 2024-11-21T00:15:33,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:33,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148133632, max timestamp=1732148133976Num uniques 859; 2024-11-21T00:15:33,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:15:33,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=35.39 KB heapSize=85.63 KB 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:33,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:33,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T00:15:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:34,000 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:34,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:34,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:34,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/b8c68cd451ac40aab906928ad5c889cf is 50, key is test_row_0/A:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:34,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741900_1076 (size=12301) 2024-11-21T00:15:34,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148134044, max timestamp=1732148134070Num uniques 859; 2024-11-21T00:15:34,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148134044, max timestamp=1732148134070Num uniques 859; 2024-11-21T00:15:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:34,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148134044, max timestamp=1732148134067Num uniques 859; 2024-11-21T00:15:34,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:34,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148134150, max timestamp=1732148134179Num uniques 1018; 2024-11-21T00:15:34,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T00:15:34,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148134150, max timestamp=1732148134179Num uniques 1018; 2024-11-21T00:15:34,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T00:15:34,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148134150, max timestamp=1732148134179Num uniques 1018; 2024-11-21T00:15:34,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T00:15:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732148134253, max timestamp=1732148134301Num uniques 1027; 2024-11-21T00:15:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T00:15:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732148134253, max timestamp=1732148134301Num uniques 1027; 2024-11-21T00:15:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T00:15:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:34,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732148134253, max timestamp=1732148134301Num uniques 1027; 2024-11-21T00:15:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T00:15:34,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1732148134387, max timestamp=1732148134432Num uniques 1036; 2024-11-21T00:15:34,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-21T00:15:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1732148134387, max timestamp=1732148134432Num uniques 1036; 2024-11-21T00:15:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-21T00:15:34,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732148134387, max timestamp=1732148134432Num uniques 1036; 2024-11-21T00:15:34,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-21T00:15:34,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/b8c68cd451ac40aab906928ad5c889cf 2024-11-21T00:15:34,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/5c557511cd72497b816b45240df7a705 is 50, key is test_row_0/B:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:34,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741901_1077 (size=12301) 2024-11-21T00:15:34,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148194546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732148134493, max timestamp=1732148134546Num uniques 1245; 2024-11-21T00:15:34,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-11-21T00:15:34,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732148134493, max timestamp=1732148134546Num uniques 1245; 2024-11-21T00:15:34,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-11-21T00:15:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732148134493, max timestamp=1732148134546Num uniques 1095; 2024-11-21T00:15:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-21T00:15:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:34,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148194716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148194720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148194724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148194847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148194847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148194848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:34,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:34,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/5c557511cd72497b816b45240df7a705 2024-11-21T00:15:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:34,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:35,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/dbee8f7101b94b4c821f26509a73ef16 is 50, key is test_row_0/C:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:35,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148195067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148195067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148195069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741902_1078 (size=12301) 2024-11-21T00:15:35,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/dbee8f7101b94b4c821f26509a73ef16 2024-11-21T00:15:35,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/b8c68cd451ac40aab906928ad5c889cf as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf 2024-11-21T00:15:35,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf, entries=150, sequenceid=325, filesize=12.0 K 2024-11-21T00:15:35,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/5c557511cd72497b816b45240df7a705 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705 2024-11-21T00:15:35,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:35,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705, entries=150, sequenceid=325, filesize=12.0 K 2024-11-21T00:15:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:35,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/dbee8f7101b94b4c821f26509a73ef16 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16 2024-11-21T00:15:35,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16, entries=150, sequenceid=325, filesize=12.0 K 2024-11-21T00:15:35,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~35.39 KB/36235, heapSize ~85.59 KB/87640, currentSize=154.31 KB/158010 for a1e0819c56248c71d5523ede3a8399ba in 1161ms, sequenceid=325, compaction requested=true 2024-11-21T00:15:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:35,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:35,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:35,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:35,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:15:35,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:35,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T00:15:35,174 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T00:15:35,176 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T00:15:35,180 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74052 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T00:15:35,180 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:35,180 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,181 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/d6db5eda52b94b0ba78fe16d8ccc87cd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=72.3 K 2024-11-21T00:15:35,182 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6db5eda52b94b0ba78fe16d8ccc87cd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:35,183 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74052 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T00:15:35,183 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:35,183 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,183 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4af2fe501eb441f697d67fd1fe58a519, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=72.3 K 2024-11-21T00:15:35,184 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc5723a5f294026b96a8a685992eb42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732148128293 2024-11-21T00:15:35,186 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af2fe501eb441f697d67fd1fe58a519, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:35,186 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76843f0dfb724a6bbd95a9b4d3c0b43c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148128998 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b15a19cbb2845e5ac33cf10cde61e00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148130207 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e39be9caf4b425db27f3807954823c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732148128293 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting efcd1788931f431286b125d7c2c51b29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148128998 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a203a2dea47b445fa9dc4955ad6f63c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732148131401 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c68cd451ac40aab906928ad5c889cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:35,187 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting be59834300b34a7abb26df51498ecc73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148130207 2024-11-21T00:15:35,188 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f3a7ccbfab2440ea6091136fad02d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732148131401 2024-11-21T00:15:35,188 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting dbee8f7101b94b4c821f26509a73ef16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:35,226 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#66 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:35,227 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/ebfbb8fc42be4a4eb312d3f1b74df6df is 50, key is test_row_0/A:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:35,237 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#67 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:35,238 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6348e1e445ac4d5390722648ee1fbf1d is 50, key is test_row_0/C:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:35,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741903_1079 (size=13051) 2024-11-21T00:15:35,291 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/ebfbb8fc42be4a4eb312d3f1b74df6df as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ebfbb8fc42be4a4eb312d3f1b74df6df 2024-11-21T00:15:35,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,292 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-21T00:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:35,303 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into ebfbb8fc42be4a4eb312d3f1b74df6df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:35,303 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:35,303 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=10, startTime=1732148135172; duration=0sec 2024-11-21T00:15:35,303 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:35,304 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:35,304 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T00:15:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/3319fb50c440401695b45a281b608f29 is 50, key is test_row_0/A:col10/1732148134717/Put/seqid=0 2024-11-21T00:15:35,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741904_1080 (size=13051) 2024-11-21T00:15:35,313 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74052 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T00:15:35,313 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:35,313 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,313 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a2ae3315bbf74c70a0fd4c35a7ebff29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=72.3 K 2024-11-21T00:15:35,316 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ae3315bbf74c70a0fd4c35a7ebff29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732148127889 2024-11-21T00:15:35,320 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ea3ca117ce4550bb8f5634a33e70e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732148128293 2024-11-21T00:15:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741905_1081 (size=12301) 2024-11-21T00:15:35,323 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3141f1b528e24e2eb6b3c0ce16a37064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148128998 2024-11-21T00:15:35,324 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6348e1e445ac4d5390722648ee1fbf1d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6348e1e445ac4d5390722648ee1fbf1d 2024-11-21T00:15:35,324 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73863e1944f4ae6b7254fd968083813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148130207 2024-11-21T00:15:35,326 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 839601a3551f4f939d05dc952d1728fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732148131401 2024-11-21T00:15:35,327 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c557511cd72497b816b45240df7a705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:35,345 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into 6348e1e445ac4d5390722648ee1fbf1d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:35,345 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:35,345 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=10, startTime=1732148135173; duration=0sec 2024-11-21T00:15:35,345 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:35,345 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:35,360 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:35,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/407c828905784a478cd4f54ee78560ab is 50, key is test_row_0/B:col10/1732148133975/Put/seqid=0 2024-11-21T00:15:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:35,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741906_1082 (size=13051) 2024-11-21T00:15:35,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148195392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148195393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148195395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148195506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148195506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148195507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148195718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148195718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148195720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/3319fb50c440401695b45a281b608f29 2024-11-21T00:15:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b06107b7fcf64c3fbe62946b70bca3df is 50, key is test_row_0/B:col10/1732148134717/Put/seqid=0 2024-11-21T00:15:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741907_1083 (size=12301) 2024-11-21T00:15:35,803 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/407c828905784a478cd4f54ee78560ab as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/407c828905784a478cd4f54ee78560ab 2024-11-21T00:15:35,808 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b06107b7fcf64c3fbe62946b70bca3df 2024-11-21T00:15:35,816 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into 407c828905784a478cd4f54ee78560ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:35,816 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:35,816 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=10, startTime=1732148135172; duration=0sec 2024-11-21T00:15:35,816 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:35,816 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d2fa93456bec4ab2befbb65b1fe54234 is 50, key is test_row_0/C:col10/1732148134717/Put/seqid=0 2024-11-21T00:15:35,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741908_1084 (size=12301) 2024-11-21T00:15:35,841 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d2fa93456bec4ab2befbb65b1fe54234 2024-11-21T00:15:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/3319fb50c440401695b45a281b608f29 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29 2024-11-21T00:15:35,866 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:15:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/b06107b7fcf64c3fbe62946b70bca3df as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df 2024-11-21T00:15:35,879 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:15:35,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/d2fa93456bec4ab2befbb65b1fe54234 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234 2024-11-21T00:15:35,894 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:15:35,901 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for a1e0819c56248c71d5523ede3a8399ba in 609ms, sequenceid=431, compaction requested=false 2024-11-21T00:15:35,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:35,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:35,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T00:15:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T00:15:35,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T00:15:35,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9030 sec 2024-11-21T00:15:35,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9120 sec 2024-11-21T00:15:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:36,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:36,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7aaafb797c6148d8bfdf6461e7dbf813 is 50, key is test_row_0/A:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:36,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741909_1085 (size=12301) 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148136064, max timestamp=1732148136084Num uniques 859; 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148136064, max timestamp=1732148136084Num uniques 859; 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148136064, max timestamp=1732148136084Num uniques 859; 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:36,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148196128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148196129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148196130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:15:36,147 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:36,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T00:15:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:36,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:36,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:36,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148196246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148196246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148196246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:36,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:36,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148196457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148196457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148196457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7aaafb797c6148d8bfdf6461e7dbf813 2024-11-21T00:15:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58e34a74ed5048a0961545019d0d82e9 is 50, key is test_row_0/B:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741910_1086 (size=12301) 2024-11-21T00:15:36,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148196766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148196768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148196777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:36,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58e34a74ed5048a0961545019d0d82e9 2024-11-21T00:15:36,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:36,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:36,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:36,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:36,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/c6cb5a1eda014007a7aaaa4fb1df33c7 is 50, key is test_row_0/C:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:37,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741911_1087 (size=12301) 2024-11-21T00:15:37,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/c6cb5a1eda014007a7aaaa4fb1df33c7 2024-11-21T00:15:37,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/7aaafb797c6148d8bfdf6461e7dbf813 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813 2024-11-21T00:15:37,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T00:15:37,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/58e34a74ed5048a0961545019d0d82e9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9 2024-11-21T00:15:37,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T00:15:37,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/c6cb5a1eda014007a7aaaa4fb1df33c7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7 2024-11-21T00:15:37,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T00:15:37,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a1e0819c56248c71d5523ede3a8399ba in 1050ms, sequenceid=444, compaction requested=true 2024-11-21T00:15:37,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:37,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:37,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:37,081 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:37,081 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:37,083 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:37,083 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:37,084 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:37,084 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:37,084 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:37,084 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:37,084 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ebfbb8fc42be4a4eb312d3f1b74df6df, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=36.8 K 2024-11-21T00:15:37,084 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/407c828905784a478cd4f54ee78560ab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=36.8 K 2024-11-21T00:15:37,085 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 407c828905784a478cd4f54ee78560ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:37,085 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfbb8fc42be4a4eb312d3f1b74df6df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:37,086 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b06107b7fcf64c3fbe62946b70bca3df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732148134709 2024-11-21T00:15:37,086 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3319fb50c440401695b45a281b608f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732148134709 2024-11-21T00:15:37,086 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e34a74ed5048a0961545019d0d82e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:37,086 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aaafb797c6148d8bfdf6461e7dbf813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:37,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T00:15:37,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:37,118 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:37,124 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:37,125 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/b613f300260e4842bf449afef8402952 is 50, key is test_row_0/A:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/92d52e149a5248d4bb6e51adc79e58d1 is 50, key is test_row_0/A:col10/1732148136087/Put/seqid=0 2024-11-21T00:15:37,134 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:37,135 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1fb06819d8954dbcbf20a9c47ea1cda2 is 50, key is test_row_0/B:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:37,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741912_1088 (size=13153) 2024-11-21T00:15:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741913_1089 (size=12301) 2024-11-21T00:15:37,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741914_1090 (size=13153) 2024-11-21T00:15:37,230 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/1fb06819d8954dbcbf20a9c47ea1cda2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1fb06819d8954dbcbf20a9c47ea1cda2 2024-11-21T00:15:37,239 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into 1fb06819d8954dbcbf20a9c47ea1cda2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:37,240 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:37,241 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=13, startTime=1732148137081; duration=0sec 2024-11-21T00:15:37,242 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:37,242 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:37,242 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:37,244 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:37,244 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:37,244 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:37,245 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6348e1e445ac4d5390722648ee1fbf1d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=36.8 K 2024-11-21T00:15:37,246 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6348e1e445ac4d5390722648ee1fbf1d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732148133972 2024-11-21T00:15:37,246 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d2fa93456bec4ab2befbb65b1fe54234, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732148134709 2024-11-21T00:15:37,257 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cb5a1eda014007a7aaaa4fb1df33c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:37,318 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:37,319 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/f33db39c7ade4045a5b05967202a6c19 is 50, key is test_row_0/C:col10/1732148135390/Put/seqid=0 2024-11-21T00:15:37,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148197354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741915_1091 (size=13153) 2024-11-21T00:15:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148197358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148197365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148197519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148197521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148197523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,602 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/b613f300260e4842bf449afef8402952 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b613f300260e4842bf449afef8402952 2024-11-21T00:15:37,613 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into b613f300260e4842bf449afef8402952(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:37,613 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:37,613 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=13, startTime=1732148137081; duration=0sec 2024-11-21T00:15:37,614 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:37,614 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:37,619 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/92d52e149a5248d4bb6e51adc79e58d1 2024-11-21T00:15:37,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/fac9bd75d3624168af49a756eb0c89be is 50, key is test_row_0/B:col10/1732148136087/Put/seqid=0 2024-11-21T00:15:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741916_1092 (size=12301) 2024-11-21T00:15:37,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/fac9bd75d3624168af49a756eb0c89be 2024-11-21T00:15:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/01dd8a9617cd4e9fafdc38deb007c2cf is 50, key is test_row_0/C:col10/1732148136087/Put/seqid=0 2024-11-21T00:15:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148197737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148197750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148197750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:37,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741917_1093 (size=12301) 2024-11-21T00:15:37,775 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/01dd8a9617cd4e9fafdc38deb007c2cf 2024-11-21T00:15:37,786 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/f33db39c7ade4045a5b05967202a6c19 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/f33db39c7ade4045a5b05967202a6c19 2024-11-21T00:15:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/92d52e149a5248d4bb6e51adc79e58d1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1 2024-11-21T00:15:37,798 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into f33db39c7ade4045a5b05967202a6c19(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:37,798 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:37,798 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=13, startTime=1732148137082; duration=0sec 2024-11-21T00:15:37,798 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:37,798 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:37,799 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T00:15:37,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/fac9bd75d3624168af49a756eb0c89be as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be 2024-11-21T00:15:37,810 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T00:15:37,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/01dd8a9617cd4e9fafdc38deb007c2cf as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf 2024-11-21T00:15:37,829 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T00:15:37,832 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for a1e0819c56248c71d5523ede3a8399ba in 714ms, sequenceid=479, compaction requested=false 2024-11-21T00:15:37,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:37,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:37,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T00:15:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T00:15:37,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T00:15:37,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6760 sec 2024-11-21T00:15:37,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6880 sec 2024-11-21T00:15:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:38,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:38,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/95e2383dd6b3426ebeeb50e38e4096d9 is 50, key is test_row_0/A:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:38,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741918_1094 (size=14741) 2024-11-21T00:15:38,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148138105, max timestamp=1732148138128Num uniques 859; 2024-11-21T00:15:38,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:38,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148138105, max timestamp=1732148138128Num uniques 859; 2024-11-21T00:15:38,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:38,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148138105, max timestamp=1732148138128Num uniques 859; 2024-11-21T00:15:38,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:38,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148198193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148198196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148198203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:15:38,307 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:38,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-21T00:15:38,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:38,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:38,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T00:15:38,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148198307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148198316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148198316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T00:15:38,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T00:15:38,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:38,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148198517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/95e2383dd6b3426ebeeb50e38e4096d9 2024-11-21T00:15:38,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148198526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148198526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:38,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/564c1fe7558a420abf56069d8da2e69b is 50, key is test_row_0/B:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:38,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741919_1095 (size=12301) 2024-11-21T00:15:38,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/564c1fe7558a420abf56069d8da2e69b 2024-11-21T00:15:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T00:15:38,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/56275e2851f1497799d13e8354bb35d4 is 50, key is test_row_0/C:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:38,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T00:15:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741920_1096 (size=12301) 2024-11-21T00:15:38,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/56275e2851f1497799d13e8354bb35d4 2024-11-21T00:15:38,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/95e2383dd6b3426ebeeb50e38e4096d9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9 2024-11-21T00:15:38,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9, entries=200, sequenceid=493, filesize=14.4 K 2024-11-21T00:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/564c1fe7558a420abf56069d8da2e69b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b 2024-11-21T00:15:38,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T00:15:38,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/56275e2851f1497799d13e8354bb35d4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4 2024-11-21T00:15:38,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T00:15:38,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a1e0819c56248c71d5523ede3a8399ba in 697ms, sequenceid=493, compaction requested=true 2024-11-21T00:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:38,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:38,752 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:38,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:38,752 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:38,754 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:38,754 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:38,754 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,754 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b613f300260e4842bf449afef8402952, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=39.3 K 2024-11-21T00:15:38,759 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:38,759 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:38,759 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,760 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1fb06819d8954dbcbf20a9c47ea1cda2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=36.9 K 2024-11-21T00:15:38,761 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b613f300260e4842bf449afef8402952, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:38,762 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb06819d8954dbcbf20a9c47ea1cda2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:38,762 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d52e149a5248d4bb6e51adc79e58d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732148136087 2024-11-21T00:15:38,763 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fac9bd75d3624168af49a756eb0c89be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732148136087 2024-11-21T00:15:38,763 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e2383dd6b3426ebeeb50e38e4096d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137315 2024-11-21T00:15:38,764 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 564c1fe7558a420abf56069d8da2e69b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137350 2024-11-21T00:15:38,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:38,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:38,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:38,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:38,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T00:15:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,797 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-21T00:15:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:38,802 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:38,803 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/ac1a01ea1da74d6f82a9b8f9672912e5 is 50, key is test_row_0/A:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/926160a9db0d4e4dab9a2cc9da8dd64b is 50, key is test_row_0/A:col10/1732148138191/Put/seqid=0 2024-11-21T00:15:38,822 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:38,823 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/d40bdca496b54e608aabd06d0afa0ece is 50, key is test_row_0/B:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:38,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. as already flushing 2024-11-21T00:15:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741921_1097 (size=13255) 2024-11-21T00:15:38,878 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/ac1a01ea1da74d6f82a9b8f9672912e5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ac1a01ea1da74d6f82a9b8f9672912e5 2024-11-21T00:15:38,893 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into ac1a01ea1da74d6f82a9b8f9672912e5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:38,893 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:38,893 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=13, startTime=1732148138752; duration=0sec 2024-11-21T00:15:38,893 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:38,893 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:38,893 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:38,896 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:38,896 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:38,896 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:38,896 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/f33db39c7ade4045a5b05967202a6c19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=36.9 K 2024-11-21T00:15:38,897 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33db39c7ade4045a5b05967202a6c19, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732148135390 2024-11-21T00:15:38,898 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01dd8a9617cd4e9fafdc38deb007c2cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732148136087 2024-11-21T00:15:38,898 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56275e2851f1497799d13e8354bb35d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137350 2024-11-21T00:15:38,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741922_1098 (size=12301) 2024-11-21T00:15:38,924 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/926160a9db0d4e4dab9a2cc9da8dd64b 2024-11-21T00:15:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741923_1099 (size=13255) 2024-11-21T00:15:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T00:15:38,970 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:38,971 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/b610b14c3b154c1cb1666ebbf86aa708 is 50, key is test_row_0/C:col10/1732148138052/Put/seqid=0 2024-11-21T00:15:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3d9a39f514d4431195f9675cd3df1d02 is 50, key is test_row_0/B:col10/1732148138191/Put/seqid=0 2024-11-21T00:15:39,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148198987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148198989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148198991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741924_1100 (size=13255) 2024-11-21T00:15:39,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741925_1101 (size=12301) 2024-11-21T00:15:39,072 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3d9a39f514d4431195f9675cd3df1d02 2024-11-21T00:15:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6bf4a41f59624ad99a1cbf68aab9a6ff is 50, key is test_row_0/C:col10/1732148138191/Put/seqid=0 2024-11-21T00:15:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49126 deadline: 1732148199117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49142 deadline: 1732148199117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732148199123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:15:39,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741926_1102 (size=12301) 2024-11-21T00:15:39,160 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6bf4a41f59624ad99a1cbf68aab9a6ff 2024-11-21T00:15:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/926160a9db0d4e4dab9a2cc9da8dd64b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b 2024-11-21T00:15:39,183 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b, entries=150, sequenceid=529, filesize=12.0 K 2024-11-21T00:15:39,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/3d9a39f514d4431195f9675cd3df1d02 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02 2024-11-21T00:15:39,194 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02, entries=150, sequenceid=529, filesize=12.0 K 2024-11-21T00:15:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/6bf4a41f59624ad99a1cbf68aab9a6ff as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff 2024-11-21T00:15:39,210 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff, entries=150, sequenceid=529, filesize=12.0 K 2024-11-21T00:15:39,211 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a1e0819c56248c71d5523ede3a8399ba in 414ms, sequenceid=529, compaction requested=false 2024-11-21T00:15:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-21T00:15:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-21T00:15:39,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T00:15:39,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 903 msec 2024-11-21T00:15:39,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 911 msec 2024-11-21T00:15:39,356 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/d40bdca496b54e608aabd06d0afa0ece as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d40bdca496b54e608aabd06d0afa0ece 2024-11-21T00:15:39,436 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into d40bdca496b54e608aabd06d0afa0ece(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:39,436 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:39,436 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=13, startTime=1732148138752; duration=0sec 2024-11-21T00:15:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:39,437 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:39,437 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:39,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:15:39,438 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T00:15:39,438 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,439 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T00:15:39,439 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:39,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:39,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:39,443 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T00:15:39,443 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,446 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,447 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T00:15:39,447 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:39,450 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,450 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,452 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,452 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,454 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/b610b14c3b154c1cb1666ebbf86aa708 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/b610b14c3b154c1cb1666ebbf86aa708 2024-11-21T00:15:39,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/6dfd0f922b63440a85072b0267e58a68 is 50, key is test_row_0/A:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:39,471 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into b610b14c3b154c1cb1666ebbf86aa708(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:39,471 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:39,471 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=13, startTime=1732148138781; duration=0sec 2024-11-21T00:15:39,472 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:39,472 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:39,473 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T00:15:39,473 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,473 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,473 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,488 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T00:15:39,488 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,491 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,492 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T00:15:39,492 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,492 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,492 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,496 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T00:15:39,496 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:39,496 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,496 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,504 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:39,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741927_1103 (size=12301) 2024-11-21T00:15:39,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/6dfd0f922b63440a85072b0267e58a68 2024-11-21T00:15:39,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/8d6e519787f74a0d88426f8d68c875dd is 50, key is test_row_0/B:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741928_1104 (size=12301) 2024-11-21T00:15:40,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/8d6e519787f74a0d88426f8d68c875dd 2024-11-21T00:15:40,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/8a49b98acf38448a9a79325c4be27994 is 50, key is test_row_0/C:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:40,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741929_1105 (size=12301) 2024-11-21T00:15:40,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/8a49b98acf38448a9a79325c4be27994 2024-11-21T00:15:40,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/6dfd0f922b63440a85072b0267e58a68 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68 2024-11-21T00:15:40,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T00:15:40,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/8d6e519787f74a0d88426f8d68c875dd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd 2024-11-21T00:15:40,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T00:15:40,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/8a49b98acf38448a9a79325c4be27994 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994 2024-11-21T00:15:40,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T00:15:40,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for a1e0819c56248c71d5523ede3a8399ba in 705ms, sequenceid=545, compaction requested=true 2024-11-21T00:15:40,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:40,144 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:40,144 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1e0819c56248c71d5523ede3a8399ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:40,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:40,145 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:40,145 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/B is initiating minor compaction (all files) 2024-11-21T00:15:40,145 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/B in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:40,145 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d40bdca496b54e608aabd06d0afa0ece, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=37.0 K 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d40bdca496b54e608aabd06d0afa0ece, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137350 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/A is initiating minor compaction (all files) 2024-11-21T00:15:40,146 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/A in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9a39f514d4431195f9675cd3df1d02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732148138146 2024-11-21T00:15:40,146 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ac1a01ea1da74d6f82a9b8f9672912e5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=37.0 K 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac1a01ea1da74d6f82a9b8f9672912e5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137350 2024-11-21T00:15:40,146 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6e519787f74a0d88426f8d68c875dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1732148138926 2024-11-21T00:15:40,147 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926160a9db0d4e4dab9a2cc9da8dd64b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732148138146 2024-11-21T00:15:40,147 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dfd0f922b63440a85072b0267e58a68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1732148138926 2024-11-21T00:15:40,158 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:40,158 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:40,158 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/9a9aa42e46284407b0089b0b6c42ed76 is 50, key is test_row_0/B:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:40,159 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/6b55fd5b069a49e0b2933edca1267d01 is 50, key is test_row_0/A:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:40,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741931_1107 (size=13357) 2024-11-21T00:15:40,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741930_1106 (size=13357) 2024-11-21T00:15:40,548 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T00:15:40,548 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:40,548 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T00:15:40,548 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,548 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:15:40,548 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,548 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,548 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:40,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2004 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1983 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 939 2024-11-21T00:15:40,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2805 rows 2024-11-21T00:15:40,550 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-11-21T00:15:40,550 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2835 rows 2024-11-21T00:15:40,550 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:15:40,550 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:15:40,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:40,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:15:40,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:15:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:40,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148140561"}]},"ts":"1732148140561"} 2024-11-21T00:15:40,564 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:15:40,564 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:15:40,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:15:40,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, UNASSIGN}] 2024-11-21T00:15:40,570 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, UNASSIGN 2024-11-21T00:15:40,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a1e0819c56248c71d5523ede3a8399ba, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:40,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, UNASSIGN because future has completed 2024-11-21T00:15:40,574 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:15:40,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:40,584 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/9a9aa42e46284407b0089b0b6c42ed76 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/9a9aa42e46284407b0089b0b6c42ed76 2024-11-21T00:15:40,591 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/B of a1e0819c56248c71d5523ede3a8399ba into 9a9aa42e46284407b0089b0b6c42ed76(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:40,591 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:40,591 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/B, priority=13, startTime=1732148140144; duration=0sec 2024-11-21T00:15:40,591 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/6b55fd5b069a49e0b2933edca1267d01 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6b55fd5b069a49e0b2933edca1267d01 2024-11-21T00:15:40,591 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:40,591 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:B 2024-11-21T00:15:40,591 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:40,592 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:40,592 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): a1e0819c56248c71d5523ede3a8399ba/C is initiating minor compaction (all files) 2024-11-21T00:15:40,592 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1e0819c56248c71d5523ede3a8399ba/C in TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:40,592 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/b610b14c3b154c1cb1666ebbf86aa708, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp, totalSize=37.0 K 2024-11-21T00:15:40,593 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b610b14c3b154c1cb1666ebbf86aa708, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732148137350 2024-11-21T00:15:40,593 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf4a41f59624ad99a1cbf68aab9a6ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732148138146 2024-11-21T00:15:40,594 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a49b98acf38448a9a79325c4be27994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1732148138926 2024-11-21T00:15:40,597 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/A of a1e0819c56248c71d5523ede3a8399ba into 6b55fd5b069a49e0b2933edca1267d01(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:40,597 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:40,597 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/A, priority=13, startTime=1732148140144; duration=0sec 2024-11-21T00:15:40,597 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:40,597 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:A 2024-11-21T00:15:40,603 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1e0819c56248c71d5523ede3a8399ba#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:40,604 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/cb2f0a4d349f46858cb437f8e1f88299 is 50, key is test_row_0/C:col10/1732148138926/Put/seqid=0 2024-11-21T00:15:40,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741932_1108 (size=13357) 2024-11-21T00:15:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:40,732 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:40,732 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:15:40,733 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing a1e0819c56248c71d5523ede3a8399ba, disabling compactions & flushes 2024-11-21T00:15:40,733 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:41,018 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/cb2f0a4d349f46858cb437f8e1f88299 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/cb2f0a4d349f46858cb437f8e1f88299 2024-11-21T00:15:41,028 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1e0819c56248c71d5523ede3a8399ba/C of a1e0819c56248c71d5523ede3a8399ba into cb2f0a4d349f46858cb437f8e1f88299(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:41,028 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1e0819c56248c71d5523ede3a8399ba: 2024-11-21T00:15:41,028 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:41,028 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., storeName=a1e0819c56248c71d5523ede3a8399ba/C, priority=13, startTime=1732148140144; duration=0sec 2024-11-21T00:15:41,028 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:41,028 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. after waiting 0 ms 2024-11-21T00:15:41,028 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:41,028 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1e0819c56248c71d5523ede3a8399ba:C 2024-11-21T00:15:41,028 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:41,028 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing a1e0819c56248c71d5523ede3a8399ba 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-21T00:15:41,028 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=A 2024-11-21T00:15:41,028 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:41,029 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=B 2024-11-21T00:15:41,029 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:41,029 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1e0819c56248c71d5523ede3a8399ba, store=C 2024-11-21T00:15:41,029 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:41,034 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/af0c06b561464ef1b099411bae342444 is 50, key is test_row_2/A:col10/1732148140547/Put/seqid=0 2024-11-21T00:15:41,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741933_1109 (size=7415) 2024-11-21T00:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:41,457 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/af0c06b561464ef1b099411bae342444 2024-11-21T00:15:41,473 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/e2458e4444474eea83ff232efec7bc4e is 50, key is test_row_2/B:col10/1732148140547/Put/seqid=0 2024-11-21T00:15:41,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741934_1110 (size=7415) 2024-11-21T00:15:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:41,882 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/e2458e4444474eea83ff232efec7bc4e 2024-11-21T00:15:41,890 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/e4bdcfec0638432799e48e39996f22ee is 50, key is test_row_2/C:col10/1732148140547/Put/seqid=0 2024-11-21T00:15:41,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741935_1111 (size=7415) 2024-11-21T00:15:42,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:15:42,296 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/e4bdcfec0638432799e48e39996f22ee 2024-11-21T00:15:42,302 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/A/af0c06b561464ef1b099411bae342444 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/af0c06b561464ef1b099411bae342444 2024-11-21T00:15:42,308 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/af0c06b561464ef1b099411bae342444, entries=50, sequenceid=554, filesize=7.2 K 2024-11-21T00:15:42,310 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/B/e2458e4444474eea83ff232efec7bc4e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/e2458e4444474eea83ff232efec7bc4e 2024-11-21T00:15:42,315 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/e2458e4444474eea83ff232efec7bc4e, entries=50, sequenceid=554, filesize=7.2 K 2024-11-21T00:15:42,316 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/.tmp/C/e4bdcfec0638432799e48e39996f22ee as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/e4bdcfec0638432799e48e39996f22ee 2024-11-21T00:15:42,321 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/e4bdcfec0638432799e48e39996f22ee, entries=50, sequenceid=554, filesize=7.2 K 2024-11-21T00:15:42,322 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for a1e0819c56248c71d5523ede3a8399ba in 1294ms, sequenceid=554, compaction requested=false 2024-11-21T00:15:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/e88ce29c33b747fb96c52567b7bf1c23, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a126ddf27fb54fd699e2dd10b82d756e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/8131a9bf5d9c4c009205db855d926194, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/d6db5eda52b94b0ba78fe16d8ccc87cd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ebfbb8fc42be4a4eb312d3f1b74df6df, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b613f300260e4842bf449afef8402952, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ac1a01ea1da74d6f82a9b8f9672912e5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68] to archive 2024-11-21T00:15:42,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:15:42,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/f12aa4db8b0e431d86cd3999a6d8d816 2024-11-21T00:15:42,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/9a8654bccb7e460e9fc1efaa1f4f6d18 2024-11-21T00:15:42,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/fb24f169bb2a49db89ebd0b1d7ff12d9 2024-11-21T00:15:42,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/075e7d59528346d599c8d05eb46dd571 2024-11-21T00:15:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/e88ce29c33b747fb96c52567b7bf1c23 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/e88ce29c33b747fb96c52567b7bf1c23 2024-11-21T00:15:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/0486aa891cb04b4da9646f13dc267b9f 2024-11-21T00:15:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/976deb4a40f1477594892c6c4d0f61a3 2024-11-21T00:15:42,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a126ddf27fb54fd699e2dd10b82d756e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a126ddf27fb54fd699e2dd10b82d756e 2024-11-21T00:15:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/edaa80ec441b43ae8fb5d9bb32b2eedb 2024-11-21T00:15:42,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7d5aa08459f143e6bcea11e7bb22b251 2024-11-21T00:15:42,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/10a73f2ab7ce478099e3d1410a71402d 2024-11-21T00:15:42,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4422d5498ec149bbbc7a70f64668befd 2024-11-21T00:15:42,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/8131a9bf5d9c4c009205db855d926194 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/8131a9bf5d9c4c009205db855d926194 2024-11-21T00:15:42,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/01888cf10ca9488ea98fe7a889c8ef17 2024-11-21T00:15:42,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/64f1b87394fc4faa9dbba1bbc74b8690 2024-11-21T00:15:42,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a13a7ea10d7d4fad9277f934b125687c 2024-11-21T00:15:42,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/d6db5eda52b94b0ba78fe16d8ccc87cd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/d6db5eda52b94b0ba78fe16d8ccc87cd 2024-11-21T00:15:42,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/bbc5723a5f294026b96a8a685992eb42 2024-11-21T00:15:42,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/76843f0dfb724a6bbd95a9b4d3c0b43c 2024-11-21T00:15:42,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/4b15a19cbb2845e5ac33cf10cde61e00 2024-11-21T00:15:42,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/a203a2dea47b445fa9dc4955ad6f63c3 2024-11-21T00:15:42,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ebfbb8fc42be4a4eb312d3f1b74df6df to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ebfbb8fc42be4a4eb312d3f1b74df6df 2024-11-21T00:15:42,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b8c68cd451ac40aab906928ad5c889cf 2024-11-21T00:15:42,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/3319fb50c440401695b45a281b608f29 2024-11-21T00:15:42,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b613f300260e4842bf449afef8402952 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/b613f300260e4842bf449afef8402952 2024-11-21T00:15:42,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/7aaafb797c6148d8bfdf6461e7dbf813 2024-11-21T00:15:42,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/92d52e149a5248d4bb6e51adc79e58d1 2024-11-21T00:15:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/95e2383dd6b3426ebeeb50e38e4096d9 2024-11-21T00:15:42,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ac1a01ea1da74d6f82a9b8f9672912e5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/ac1a01ea1da74d6f82a9b8f9672912e5 2024-11-21T00:15:42,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/926160a9db0d4e4dab9a2cc9da8dd64b 2024-11-21T00:15:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6dfd0f922b63440a85072b0267e58a68 2024-11-21T00:15:42,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/6e7c4b30d53c4cd89512fba931596ee9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/73d460b96aa644d58d62cc5648360603, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/bd001632ceb54ce8b4d17bc4729a3071, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a2ae3315bbf74c70a0fd4c35a7ebff29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/407c828905784a478cd4f54ee78560ab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1fb06819d8954dbcbf20a9c47ea1cda2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d40bdca496b54e608aabd06d0afa0ece, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd] to archive 2024-11-21T00:15:42,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:15:42,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/61907d788f6c4e92b104c0e80fa02d54 2024-11-21T00:15:42,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/25a459e0a6dc464ca5242def6a3e872c 2024-11-21T00:15:42,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/ec28a4540a844d6797a8347a8a0737f8 2024-11-21T00:15:42,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/6e7c4b30d53c4cd89512fba931596ee9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/6e7c4b30d53c4cd89512fba931596ee9 2024-11-21T00:15:42,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/0e2a68cdd21c42289a10672cec0df00b 2024-11-21T00:15:42,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/56ea6f5532014405b23e9b6e610a6218 2024-11-21T00:15:42,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1a05a38de7ea4a3f971a75a33a1b276c 2024-11-21T00:15:42,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/73d460b96aa644d58d62cc5648360603 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/73d460b96aa644d58d62cc5648360603 2024-11-21T00:15:42,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/f95420dc119245e2b7f20c25fed6787c 2024-11-21T00:15:42,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d8cb5e6b646040b194201d8e34ace999 2024-11-21T00:15:42,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b512c5e1663546eeb2f1eaf3948ac278 2024-11-21T00:15:42,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/bd001632ceb54ce8b4d17bc4729a3071 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/bd001632ceb54ce8b4d17bc4729a3071 2024-11-21T00:15:42,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1e0e7a1265bc41389caa14ccdb27073b 2024-11-21T00:15:42,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c6682fecfef5441f8a65732415a267a0 2024-11-21T00:15:42,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58fc345bf35a49a9bc7ba898fd54ded4 2024-11-21T00:15:42,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a2ae3315bbf74c70a0fd4c35a7ebff29 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a2ae3315bbf74c70a0fd4c35a7ebff29 2024-11-21T00:15:42,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3ffa36fe20124540bcf8a2a891b8a5cd 2024-11-21T00:15:42,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/a1ea3ca117ce4550bb8f5634a33e70e2 2024-11-21T00:15:42,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3141f1b528e24e2eb6b3c0ce16a37064 2024-11-21T00:15:42,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/c73863e1944f4ae6b7254fd968083813 2024-11-21T00:15:42,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/839601a3551f4f939d05dc952d1728fc 2024-11-21T00:15:42,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/407c828905784a478cd4f54ee78560ab to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/407c828905784a478cd4f54ee78560ab 2024-11-21T00:15:42,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/5c557511cd72497b816b45240df7a705 2024-11-21T00:15:42,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/b06107b7fcf64c3fbe62946b70bca3df 2024-11-21T00:15:42,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1fb06819d8954dbcbf20a9c47ea1cda2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/1fb06819d8954dbcbf20a9c47ea1cda2 2024-11-21T00:15:42,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/58e34a74ed5048a0961545019d0d82e9 2024-11-21T00:15:42,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/fac9bd75d3624168af49a756eb0c89be 2024-11-21T00:15:42,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d40bdca496b54e608aabd06d0afa0ece to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/d40bdca496b54e608aabd06d0afa0ece 2024-11-21T00:15:42,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/564c1fe7558a420abf56069d8da2e69b 2024-11-21T00:15:42,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/3d9a39f514d4431195f9675cd3df1d02 2024-11-21T00:15:42,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/8d6e519787f74a0d88426f8d68c875dd 2024-11-21T00:15:42,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/75e8492c32a44d09ad2b50ef4b287bc4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/31c4621140d6455d8059167297cb8106, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/1ef1eae5a9f940e4b776c59598e00d68, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4af2fe501eb441f697d67fd1fe58a519, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6348e1e445ac4d5390722648ee1fbf1d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/f33db39c7ade4045a5b05967202a6c19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/b610b14c3b154c1cb1666ebbf86aa708, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994] to archive 2024-11-21T00:15:42,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:15:42,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/033a0d8ece22451fb241805c27303104 2024-11-21T00:15:42,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ceb98e1512c84622b6763613553f139c 2024-11-21T00:15:42,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d45bd7f3db09476e8f6dc0282ed82398 2024-11-21T00:15:42,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/75e8492c32a44d09ad2b50ef4b287bc4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/75e8492c32a44d09ad2b50ef4b287bc4 2024-11-21T00:15:42,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/40531c8858df404cb73bb7d1b1319c5d 2024-11-21T00:15:42,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/fa4dcb517ad9481ea9beb0c6c6cba9ec 2024-11-21T00:15:42,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/5d2e274de3f045b5905b8f74ff2247f8 2024-11-21T00:15:42,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/31c4621140d6455d8059167297cb8106 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/31c4621140d6455d8059167297cb8106 2024-11-21T00:15:42,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/481105742c8d48ddae719a9471f6e8b0 2024-11-21T00:15:42,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba72bd09f1984506a44d6107fc86327c 2024-11-21T00:15:42,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/ba9b24e774f04e3f82ef700f7c83b6f7 2024-11-21T00:15:42,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/1ef1eae5a9f940e4b776c59598e00d68 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/1ef1eae5a9f940e4b776c59598e00d68 2024-11-21T00:15:42,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6a2b824ac2ea40379f935499e2e41f07 2024-11-21T00:15:42,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4ac0b4bb65c84a69b96b09db7e4adebc 2024-11-21T00:15:42,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3d9d15ea8eaa46e3b5f9b04124560324 2024-11-21T00:15:42,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4af2fe501eb441f697d67fd1fe58a519 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/4af2fe501eb441f697d67fd1fe58a519 2024-11-21T00:15:42,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d9e248c0fa2745d2a61bab6103d76e4d 2024-11-21T00:15:42,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6e39be9caf4b425db27f3807954823c1 2024-11-21T00:15:42,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/efcd1788931f431286b125d7c2c51b29 2024-11-21T00:15:42,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/be59834300b34a7abb26df51498ecc73 2024-11-21T00:15:42,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/3f3a7ccbfab2440ea6091136fad02d79 2024-11-21T00:15:42,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6348e1e445ac4d5390722648ee1fbf1d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6348e1e445ac4d5390722648ee1fbf1d 2024-11-21T00:15:42,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/dbee8f7101b94b4c821f26509a73ef16 2024-11-21T00:15:42,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/d2fa93456bec4ab2befbb65b1fe54234 2024-11-21T00:15:42,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/f33db39c7ade4045a5b05967202a6c19 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/f33db39c7ade4045a5b05967202a6c19 2024-11-21T00:15:42,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/c6cb5a1eda014007a7aaaa4fb1df33c7 2024-11-21T00:15:42,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/01dd8a9617cd4e9fafdc38deb007c2cf 2024-11-21T00:15:42,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/b610b14c3b154c1cb1666ebbf86aa708 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/b610b14c3b154c1cb1666ebbf86aa708 2024-11-21T00:15:42,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/56275e2851f1497799d13e8354bb35d4 2024-11-21T00:15:42,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/6bf4a41f59624ad99a1cbf68aab9a6ff 2024-11-21T00:15:42,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/8a49b98acf38448a9a79325c4be27994 2024-11-21T00:15:42,540 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/recovered.edits/557.seqid, newMaxSeqId=557, maxSeqId=1 2024-11-21T00:15:42,541 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. 2024-11-21T00:15:42,542 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for a1e0819c56248c71d5523ede3a8399ba: Waiting for close lock at 1732148140733Running coprocessor pre-close hooks at 1732148140733Disabling compacts and flushes for region at 1732148140733Disabling writes for close at 1732148141028 (+295 ms)Obtaining lock to block concurrent updates at 1732148141028Preparing flush snapshotting stores in a1e0819c56248c71d5523ede3a8399ba at 1732148141028Finished memstore snapshotting TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732148141029 (+1 ms)Flushing stores of TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba. at 1732148141030 (+1 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/A: creating writer at 1732148141030Flushing a1e0819c56248c71d5523ede3a8399ba/A: appending metadata at 1732148141033 (+3 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/A: closing flushed file at 1732148141034 (+1 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/B: creating writer at 1732148141468 (+434 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/B: appending metadata at 1732148141472 (+4 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/B: closing flushed file at 1732148141472Flushing a1e0819c56248c71d5523ede3a8399ba/C: creating writer at 1732148141887 (+415 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/C: appending metadata at 1732148141890 (+3 ms)Flushing a1e0819c56248c71d5523ede3a8399ba/C: closing flushed file at 1732148141890Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68f8a3a: reopening flushed file at 1732148142301 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4121ec1e: reopening flushed file at 1732148142309 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e8bbd3f: reopening flushed file at 1732148142315 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for a1e0819c56248c71d5523ede3a8399ba in 1294ms, sequenceid=554, compaction requested=false at 1732148142322 (+7 ms)Writing region close event to WAL at 1732148142529 (+207 ms)Running coprocessor post-close hooks at 1732148142541 (+12 ms)Closed at 1732148142541 2024-11-21T00:15:42,544 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:42,545 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a1e0819c56248c71d5523ede3a8399ba, regionState=CLOSED 2024-11-21T00:15:42,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:15:42,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T00:15:42,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure a1e0819c56248c71d5523ede3a8399ba, server=5ed4808ef0e6,40705,1732148113864 in 1.9740 sec 2024-11-21T00:15:42,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T00:15:42,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1e0819c56248c71d5523ede3a8399ba, UNASSIGN in 1.9820 sec 2024-11-21T00:15:42,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T00:15:42,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9890 sec 2024-11-21T00:15:42,558 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148142558"}]},"ts":"1732148142558"} 2024-11-21T00:15:42,560 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:15:42,560 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:15:42,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0050 sec 2024-11-21T00:15:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:15:42,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:42,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:15:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,715 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T00:15:42,717 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,723 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:42,728 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/recovered.edits] 2024-11-21T00:15:42,735 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6b55fd5b069a49e0b2933edca1267d01 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/6b55fd5b069a49e0b2933edca1267d01 2024-11-21T00:15:42,736 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/af0c06b561464ef1b099411bae342444 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/A/af0c06b561464ef1b099411bae342444 2024-11-21T00:15:42,745 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/9a9aa42e46284407b0089b0b6c42ed76 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/9a9aa42e46284407b0089b0b6c42ed76 2024-11-21T00:15:42,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/e2458e4444474eea83ff232efec7bc4e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/B/e2458e4444474eea83ff232efec7bc4e 2024-11-21T00:15:42,752 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/cb2f0a4d349f46858cb437f8e1f88299 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/cb2f0a4d349f46858cb437f8e1f88299 2024-11-21T00:15:42,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/e4bdcfec0638432799e48e39996f22ee to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/C/e4bdcfec0638432799e48e39996f22ee 2024-11-21T00:15:42,758 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/recovered.edits/557.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba/recovered.edits/557.seqid 2024-11-21T00:15:42,759 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/a1e0819c56248c71d5523ede3a8399ba 2024-11-21T00:15:42,759 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:15:42,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T00:15:42,782 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:15:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T00:15:42,827 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:15:42,830 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,830 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:15:42,830 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148142830"}]},"ts":"9223372036854775807"} 2024-11-21T00:15:42,835 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:15:42,835 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a1e0819c56248c71d5523ede3a8399ba, NAME => 'TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:15:42,836 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:15:42,836 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148142836"}]},"ts":"9223372036854775807"} 2024-11-21T00:15:42,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:15:42,841 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:42,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 130 msec 2024-11-21T00:15:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T00:15:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:15:43,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:43,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_823068347_22 at /127.0.0.1:39916 [Waiting for operation #230] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_823068347_22 at /127.0.0.1:42162 [Waiting for operation #171] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1977616121_22 at /127.0.0.1:40044 [Waiting for operation #213] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5ed4808ef0e6:40705-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 820) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1846 (was 1374) - AvailableMemoryMB LEAK? - 2024-11-21T00:15:43,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=1842 2024-11-21T00:15:43,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:15:43,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:15:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:43,071 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:15:43,072 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:43,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-21T00:15:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T00:15:43,073 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:15:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741936_1112 (size=1061) 2024-11-21T00:15:43,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:15:43,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741937_1113 (size=53) 2024-11-21T00:15:43,106 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:43,106 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cb31c4d3052e5d7cdaabfc9b52a02c70, disabling compactions & flushes 2024-11-21T00:15:43,107 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. after waiting 0 ms 2024-11-21T00:15:43,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,107 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb31c4d3052e5d7cdaabfc9b52a02c70: Waiting for close lock at 1732148143106Disabling compacts and flushes for region at 1732148143106Disabling writes for close at 1732148143107 (+1 ms)Writing region close event to WAL at 1732148143107Closed at 1732148143107 2024-11-21T00:15:43,108 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:15:43,109 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148143108"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148143108"}]},"ts":"1732148143108"} 2024-11-21T00:15:43,111 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:15:43,112 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:15:43,113 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148143112"}]},"ts":"1732148143112"} 2024-11-21T00:15:43,115 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:15:43,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, ASSIGN}] 2024-11-21T00:15:43,117 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, ASSIGN 2024-11-21T00:15:43,118 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:15:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T00:15:43,269 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:43,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, ASSIGN because future has completed 2024-11-21T00:15:43,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T00:15:43,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T00:15:43,433 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,433 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:15:43,433 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,434 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:43,434 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,434 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,435 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,437 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:43,438 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName A 2024-11-21T00:15:43,438 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:43,439 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:43,439 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,441 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:43,441 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName B 2024-11-21T00:15:43,441 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:43,442 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:43,442 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,444 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:43,444 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName C 2024-11-21T00:15:43,444 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:43,445 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:43,445 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,445 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,446 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,446 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,448 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,448 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,449 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:15:43,451 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,457 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:15:43,458 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened cb31c4d3052e5d7cdaabfc9b52a02c70; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71836929, jitterRate=0.07045365869998932}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:15:43,458 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,459 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for cb31c4d3052e5d7cdaabfc9b52a02c70: Running coprocessor pre-open hook at 1732148143434Writing region info on filesystem at 1732148143434Initializing all the Stores at 1732148143435 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148143435Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148143435Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148143435Cleaning up temporary data from old regions at 1732148143448 (+13 ms)Running coprocessor post-open hooks at 1732148143458 (+10 ms)Region opened successfully at 1732148143459 (+1 ms) 2024-11-21T00:15:43,460 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., pid=34, masterSystemTime=1732148143429 2024-11-21T00:15:43,464 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,464 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,465 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:43,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:15:43,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T00:15:43,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 in 205 msec 2024-11-21T00:15:43,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T00:15:43,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, ASSIGN in 367 msec 2024-11-21T00:15:43,493 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:15:43,493 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148143493"}]},"ts":"1732148143493"} 2024-11-21T00:15:43,495 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:15:43,496 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:15:43,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 427 msec 2024-11-21T00:15:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T00:15:43,697 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:43,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:43,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:43,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:43,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b8c94f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:43,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1c755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:43,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:43,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:43,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:15:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:15:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ad5e6af 2024-11-21T00:15:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:15:43,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:15:43,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:15:43,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:15:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:15:43,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741938_1114 (size=1097) 2024-11-21T00:15:43,740 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T00:15:43,740 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T00:15:43,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:15:43,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE}] 2024-11-21T00:15:43,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE 2024-11-21T00:15:43,752 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:43,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE because future has completed 2024-11-21T00:15:43,754 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:15:43,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:43,907 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:15:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing cb31c4d3052e5d7cdaabfc9b52a02c70, disabling compactions & flushes 2024-11-21T00:15:43,907 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. after waiting 0 ms 2024-11-21T00:15:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,911 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:15:43,912 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:43,912 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for cb31c4d3052e5d7cdaabfc9b52a02c70: Waiting for close lock at 1732148143907Running coprocessor pre-close hooks at 1732148143907Disabling compacts and flushes for region at 1732148143907Disabling writes for close at 1732148143907Writing region close event to WAL at 1732148143908 (+1 ms)Running coprocessor post-close hooks at 1732148143912 (+4 ms)Closed at 1732148143912 2024-11-21T00:15:43,912 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: cb31c4d3052e5d7cdaabfc9b52a02c70 to self. 2024-11-21T00:15:43,915 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:43,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=CLOSED 2024-11-21T00:15:43,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:15:43,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-21T00:15:43,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 in 166 msec 2024-11-21T00:15:43,923 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=true 2024-11-21T00:15:44,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE because future has completed 2024-11-21T00:15:44,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:15:44,230 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,230 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:15:44,231 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,231 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:15:44,231 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,231 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,236 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,237 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:44,241 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName A 2024-11-21T00:15:44,243 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:44,244 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:44,244 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,245 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:44,245 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName B 2024-11-21T00:15:44,245 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:44,246 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:44,246 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,246 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:15:44,247 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb31c4d3052e5d7cdaabfc9b52a02c70 columnFamilyName C 2024-11-21T00:15:44,247 DEBUG [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:44,247 INFO [StoreOpener-cb31c4d3052e5d7cdaabfc9b52a02c70-1 {}] regionserver.HStore(327): Store=cb31c4d3052e5d7cdaabfc9b52a02c70/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:15:44,247 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,247 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,248 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,250 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,250 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,250 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,251 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:15:44,252 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,253 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cb31c4d3052e5d7cdaabfc9b52a02c70; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65442749, jitterRate=-0.024827048182487488}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:15:44,253 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,254 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cb31c4d3052e5d7cdaabfc9b52a02c70: Running coprocessor pre-open hook at 1732148144231Writing region info on filesystem at 1732148144231Initializing all the Stores at 1732148144232 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148144232Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148144232Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148144232Cleaning up temporary data from old regions at 1732148144251 (+19 ms)Running coprocessor post-open hooks at 1732148144253 (+2 ms)Region opened successfully at 1732148144254 (+1 ms) 2024-11-21T00:15:44,254 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., pid=39, masterSystemTime=1732148144227 2024-11-21T00:15:44,258 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,258 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,258 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:15:44,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-21T00:15:44,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 in 186 msec 2024-11-21T00:15:44,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T00:15:44,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, REOPEN/MOVE in 515 msec 2024-11-21T00:15:44,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T00:15:44,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-21T00:15:44,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 554 msec 2024-11-21T00:15:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T00:15:44,282 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:44,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,285 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2253fa83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,287 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f90a6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457b918e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,302 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479dbdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135758f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ccdbf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5711091d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,314 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bb6d60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5830cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd06682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef4b7d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33950ad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621343d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acf3db6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a8f698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:15:44,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:15:44,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:15:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:15:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:15:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c4f6ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:15:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:15:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:15:44,346 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:15:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:15:44,350 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,357 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,371 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,372 DEBUG [Thread-591 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,372 DEBUG [Thread-591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:15:44,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:44,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:44,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:44,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:44,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:44,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,381 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T00:15:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:44,392 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:44,393 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148204406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,407 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732148144378, max timestamp=1732148144402Num uniques 859; 2024-11-21T00:15:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-21T00:15:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732148144378, max timestamp=1732148144402Num uniques 859; 2024-11-21T00:15:44,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1732148144378, max timestamp=1732148144402Num uniques 859; 2024-11-21T00:15:44,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-21T00:15:44,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-11-21T00:15:44,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,414 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,416 DEBUG [Thread-587 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,416 DEBUG [Thread-587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,419 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,420 DEBUG [Thread-602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,420 DEBUG [Thread-602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,426 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:15:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:15:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:15:44,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:15:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148204427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148204430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148204433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:15:44,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213fafeaaee41f4e6bba050c3f4e720aeb_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148144375/Put/seqid=0 2024-11-21T00:15:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148204446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741939_1115 (size=12154) 2024-11-21T00:15:44,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:44,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213fafeaaee41f4e6bba050c3f4e720aeb_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213fafeaaee41f4e6bba050c3f4e720aeb_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:44,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/bac67a48381341ce8211b5b2d1b83bdd, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:44,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/bac67a48381341ce8211b5b2d1b83bdd is 175, key is test_row_0/A:col10/1732148144375/Put/seqid=0 2024-11-21T00:15:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148204517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148204546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148204546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148204547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741940_1116 (size=30955) 2024-11-21T00:15:44,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:44,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:44,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/bac67a48381341ce8211b5b2d1b83bdd 2024-11-21T00:15:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148204578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/73824e24c1864ad88450cbdb0920eed3 is 50, key is test_row_0/B:col10/1732148144375/Put/seqid=0 2024-11-21T00:15:44,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741941_1117 (size=12001) 2024-11-21T00:15:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:44,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148204746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148204757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148204759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148204768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148204797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:44,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:44,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:44,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:44,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:45,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:45,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148205069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148205076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/73824e24c1864ad88450cbdb0920eed3 2024-11-21T00:15:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148205087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148205087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148205106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f294d6e9e7a3481899d48de10ec0ae51 is 50, key is test_row_0/C:col10/1732148144375/Put/seqid=0 2024-11-21T00:15:45,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741942_1118 (size=12001) 2024-11-21T00:15:45,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f294d6e9e7a3481899d48de10ec0ae51 2024-11-21T00:15:45,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:45,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/bac67a48381341ce8211b5b2d1b83bdd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd 2024-11-21T00:15:45,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd, entries=150, sequenceid=14, filesize=30.2 K 2024-11-21T00:15:45,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/73824e24c1864ad88450cbdb0920eed3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3 2024-11-21T00:15:45,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T00:15:45,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f294d6e9e7a3481899d48de10ec0ae51 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51 2024-11-21T00:15:45,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T00:15:45,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 919ms, sequenceid=14, compaction requested=false 2024-11-21T00:15:45,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:45,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:15:45,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=154.31 KB heapSize=388.50 KB 2024-11-21T00:15:45,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112130ae2a34a0fc4e748164f0ac3ed7c13a_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148144402/Put/seqid=0 2024-11-21T00:15:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741943_1119 (size=12154) 2024-11-21T00:15:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:45,459 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112130ae2a34a0fc4e748164f0ac3ed7c13a_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130ae2a34a0fc4e748164f0ac3ed7c13a_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:45,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a86a265e80d44e0fba6143e787011e8b, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:45,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a86a265e80d44e0fba6143e787011e8b is 175, key is test_row_0/A:col10/1732148144402/Put/seqid=0 2024-11-21T00:15:45,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741944_1120 (size=30955) 2024-11-21T00:15:45,511 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a86a265e80d44e0fba6143e787011e8b 2024-11-21T00:15:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a07b28985384354a325c3d173ad8653 is 50, key is test_row_0/B:col10/1732148144402/Put/seqid=0 2024-11-21T00:15:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:45,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741945_1121 (size=12001) 2024-11-21T00:15:45,560 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a07b28985384354a325c3d173ad8653 2024-11-21T00:15:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/9169e5a5ece8441f9fe47aa8a7c09f3f is 50, key is test_row_0/C:col10/1732148144402/Put/seqid=0 2024-11-21T00:15:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:45,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741946_1122 (size=12001) 2024-11-21T00:15:45,620 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/9169e5a5ece8441f9fe47aa8a7c09f3f 2024-11-21T00:15:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148205616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a86a265e80d44e0fba6143e787011e8b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b 2024-11-21T00:15:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148205627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148205628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148205627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148205629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,647 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b, entries=150, sequenceid=42, filesize=30.2 K 2024-11-21T00:15:45,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a07b28985384354a325c3d173ad8653 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653 2024-11-21T00:15:45,659 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T00:15:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/9169e5a5ece8441f9fe47aa8a7c09f3f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f 2024-11-21T00:15:45,678 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T00:15:45,680 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~388.45 KB/397776, currentSize=53.67 KB/54960 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 323ms, sequenceid=42, compaction requested=false 2024-11-21T00:15:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T00:15:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T00:15:45,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T00:15:45,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-11-21T00:15:45,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3040 sec 2024-11-21T00:15:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:45,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:45,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112188e01bdffd7f48e081963904176b9a02_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:45,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741947_1123 (size=14594) 2024-11-21T00:15:45,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:45,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112188e01bdffd7f48e081963904176b9a02_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188e01bdffd7f48e081963904176b9a02_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:45,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a95172eb212149999dbf593a363ca4e7, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:45,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a95172eb212149999dbf593a363ca4e7 is 175, key is test_row_0/A:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:45,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741948_1124 (size=39549) 2024-11-21T00:15:45,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a95172eb212149999dbf593a363ca4e7 2024-11-21T00:15:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148205932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/2d8a51705c374cc99ae59dead8e4bc57 is 50, key is test_row_0/B:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:45,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148205941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148205943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:45,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741949_1125 (size=12001) 2024-11-21T00:15:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148206046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148206057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148206057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148206057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148206257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148206277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148206277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148206277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/2d8a51705c374cc99ae59dead8e4bc57 2024-11-21T00:15:46,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/b82c5788682c4b29a617510e1c446348 is 50, key is test_row_0/C:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:46,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741950_1126 (size=12001) 2024-11-21T00:15:46,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/b82c5788682c4b29a617510e1c446348 2024-11-21T00:15:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/a95172eb212149999dbf593a363ca4e7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7 2024-11-21T00:15:46,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7, entries=200, sequenceid=56, filesize=38.6 K 2024-11-21T00:15:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/2d8a51705c374cc99ae59dead8e4bc57 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57 2024-11-21T00:15:46,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57, entries=150, sequenceid=56, filesize=11.7 K 2024-11-21T00:15:46,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/b82c5788682c4b29a617510e1c446348 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348 2024-11-21T00:15:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:15:46,537 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:46,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:46,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348, entries=150, sequenceid=56, filesize=11.7 K 2024-11-21T00:15:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T00:15:46,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 783ms, sequenceid=56, compaction requested=true 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:46,543 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:15:46,543 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:46,544 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:46,544 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:46,544 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:46,544 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:46,544 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,544 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,545 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=35.2 K 2024-11-21T00:15:46,545 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=99.1 K 2024-11-21T00:15:46,545 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,545 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7] 2024-11-21T00:15:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:46,545 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 73824e24c1864ad88450cbdb0920eed3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148144373 2024-11-21T00:15:46,546 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bac67a48381341ce8211b5b2d1b83bdd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148144373 2024-11-21T00:15:46,546 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a07b28985384354a325c3d173ad8653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148144402 2024-11-21T00:15:46,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:46,549 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d8a51705c374cc99ae59dead8e4bc57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:46,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:46,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:46,550 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a86a265e80d44e0fba6143e787011e8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148144402 2024-11-21T00:15:46,551 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95172eb212149999dbf593a363ca4e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:46,581 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#108 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:46,582 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/6bc7f0bb1f5e43d1911c18460701eb2b is 50, key is test_row_0/B:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:46,585 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:46,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148145929, max timestamp=1732148146583Num uniques 859; 2024-11-21T00:15:46,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148145929, max timestamp=1732148146583Num uniques 859; 2024-11-21T00:15:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:46,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148145929, max timestamp=1732148146583Num uniques 859; 2024-11-21T00:15:46,601 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411213285e4c70edd460f9d086db926f48fc0_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:46,606 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411213285e4c70edd460f9d086db926f48fc0_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:46,606 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213285e4c70edd460f9d086db926f48fc0_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:46,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:46,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=40.25 KB heapSize=106.08 KB 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:46,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741951_1127 (size=12104) 2024-11-21T00:15:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:46,661 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/6bc7f0bb1f5e43d1911c18460701eb2b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/6bc7f0bb1f5e43d1911c18460701eb2b 2024-11-21T00:15:46,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213c642ff542ab4107b609f52a2c99f141_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148146601/Put/seqid=0 2024-11-21T00:15:46,692 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into 6bc7f0bb1f5e43d1911c18460701eb2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:46,692 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:46,693 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148146543; duration=0sec 2024-11-21T00:15:46,693 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:46,693 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:15:46,693 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:46,694 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:46,694 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:15:46,695 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,695 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=35.2 K 2024-11-21T00:15:46,695 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f294d6e9e7a3481899d48de10ec0ae51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148144373 2024-11-21T00:15:46,696 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9169e5a5ece8441f9fe47aa8a7c09f3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148144402 2024-11-21T00:15:46,698 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b82c5788682c4b29a617510e1c446348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:46,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741952_1128 (size=4469) 2024-11-21T00:15:46,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:15:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741953_1129 (size=17034) 2024-11-21T00:15:46,710 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#109 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:46,711 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/4845c8aab8ec47bbb0b3bcdcc7513fb2 is 175, key is test_row_0/A:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:46,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:46,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213c642ff542ab4107b609f52a2c99f141_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213c642ff542ab4107b609f52a2c99f141_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:46,733 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#111 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:46,734 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a0220edf0867478f9bfeaa90fb39cf06 is 50, key is test_row_0/C:col10/1732148145619/Put/seqid=0 2024-11-21T00:15:46,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/dc7fbbccf11848a2b2b07617cc38c5fd, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:46,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/dc7fbbccf11848a2b2b07617cc38c5fd is 175, key is test_row_0/A:col10/1732148146601/Put/seqid=0 2024-11-21T00:15:46,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741954_1130 (size=31058) 2024-11-21T00:15:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148146736, max timestamp=1732148146786Num uniques 859; 2024-11-21T00:15:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:46,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148146736, max timestamp=1732148146786Num uniques 859; 2024-11-21T00:15:46,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:46,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148146736, max timestamp=1732148146786Num uniques 859; 2024-11-21T00:15:46,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:46,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741956_1132 (size=48139) 2024-11-21T00:15:46,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/dc7fbbccf11848a2b2b07617cc38c5fd 2024-11-21T00:15:46,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741955_1131 (size=12104) 2024-11-21T00:15:46,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/f790ee3366b148259f8a2e7288c47fb2 is 50, key is test_row_0/B:col10/1732148146601/Put/seqid=0 2024-11-21T00:15:46,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a0220edf0867478f9bfeaa90fb39cf06 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a0220edf0867478f9bfeaa90fb39cf06 2024-11-21T00:15:46,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:15:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:46,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741957_1133 (size=12001) 2024-11-21T00:15:46,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/f790ee3366b148259f8a2e7288c47fb2 2024-11-21T00:15:46,897 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into a0220edf0867478f9bfeaa90fb39cf06(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:46,897 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:46,897 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148146543; duration=0sec 2024-11-21T00:15:46,897 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:46,897 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:15:46,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/065dfd09ccd3450985b606f14f6ce0d8 is 50, key is test_row_0/C:col10/1732148146601/Put/seqid=0 2024-11-21T00:15:46,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148206932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148206936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148206943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148206952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741958_1134 (size=12001) 2024-11-21T00:15:46,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/065dfd09ccd3450985b606f14f6ce0d8 2024-11-21T00:15:46,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148206957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/dc7fbbccf11848a2b2b07617cc38c5fd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd 2024-11-21T00:15:46,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd, entries=250, sequenceid=80, filesize=47.0 K 2024-11-21T00:15:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/f790ee3366b148259f8a2e7288c47fb2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2 2024-11-21T00:15:47,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T00:15:47,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/065dfd09ccd3450985b606f14f6ce0d8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8 2024-11-21T00:15:47,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T00:15:47,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=140.89 KB/144270 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 413ms, sequenceid=80, compaction requested=false 2024-11-21T00:15:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:47,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:15:47,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:47,023 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-21T00:15:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215c6ea8d155e54e65bd997eb103ce4847_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148146943/Put/seqid=0 2024-11-21T00:15:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:47,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741959_1135 (size=12154) 2024-11-21T00:15:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148207096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148207097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148207101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148207103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148207109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,165 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/4845c8aab8ec47bbb0b3bcdcc7513fb2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2 2024-11-21T00:15:47,174 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 4845c8aab8ec47bbb0b3bcdcc7513fb2(size=30.3 K), total size for store is 77.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:47,174 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:47,174 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148146543; duration=0sec 2024-11-21T00:15:47,174 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:47,174 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:15:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:47,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148207227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148207227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148207227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148207227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148207227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148207437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148207442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148207447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148207447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148207447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:47,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215c6ea8d155e54e65bd997eb103ce4847_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c6ea8d155e54e65bd997eb103ce4847_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:47,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6a651a7c7dc242628d956f458721547f, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:47,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6a651a7c7dc242628d956f458721547f is 175, key is test_row_0/A:col10/1732148146943/Put/seqid=0 2024-11-21T00:15:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741960_1136 (size=30955) 2024-11-21T00:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148207757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148207758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148207758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148207759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148207767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:47,998 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6a651a7c7dc242628d956f458721547f 2024-11-21T00:15:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7745c76b25a04823b81c4fddd3c3fa27 is 50, key is test_row_0/B:col10/1732148146943/Put/seqid=0 2024-11-21T00:15:48,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741961_1137 (size=12001) 2024-11-21T00:15:48,071 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7745c76b25a04823b81c4fddd3c3fa27 2024-11-21T00:15:48,081 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:15:48,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/dca611da13bd47cfa5a942c1f8e9785e is 50, key is test_row_0/C:col10/1732148146943/Put/seqid=0 2024-11-21T00:15:48,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741962_1138 (size=12001) 2024-11-21T00:15:48,155 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/dca611da13bd47cfa5a942c1f8e9785e 2024-11-21T00:15:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6a651a7c7dc242628d956f458721547f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f 2024-11-21T00:15:48,169 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f, entries=150, sequenceid=118, filesize=30.2 K 2024-11-21T00:15:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7745c76b25a04823b81c4fddd3c3fa27 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27 2024-11-21T00:15:48,185 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:15:48,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/dca611da13bd47cfa5a942c1f8e9785e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e 2024-11-21T00:15:48,212 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:15:48,214 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1191ms, sequenceid=118, compaction requested=true 2024-11-21T00:15:48,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:48,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:48,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T00:15:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T00:15:48,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T00:15:48,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2024-11-21T00:15:48,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6840 sec 2024-11-21T00:15:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:48,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:48,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abc83e88b7e246da862981e06f188891_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:48,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741963_1139 (size=17184) 2024-11-21T00:15:48,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148208309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148208312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148208316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148208319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148208321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abc83e88b7e246da862981e06f188891_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc83e88b7e246da862981e06f188891_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:48,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/30b00ef46e9c4904a566892a438c0bfd, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:48,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/30b00ef46e9c4904a566892a438c0bfd is 175, key is test_row_0/A:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:48,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741964_1140 (size=48289) 2024-11-21T00:15:48,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/30b00ef46e9c4904a566892a438c0bfd 2024-11-21T00:15:48,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/eb7ac8634bc246e38afd72e055c83ad2 is 50, key is test_row_0/B:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741965_1141 (size=12101) 2024-11-21T00:15:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148208426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148208426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148208437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148208437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148208440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148208636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148208636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148208647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148208647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148208647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:15:48,697 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:48,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T00:15:48,699 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:48,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:48,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:48,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/eb7ac8634bc246e38afd72e055c83ad2 2024-11-21T00:15:48,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/232c221d26ab4eb1b75bf753233ba275 is 50, key is test_row_0/C:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:48,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:15:48,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:48,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:48,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:48,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741966_1142 (size=12101) 2024-11-21T00:15:48,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148208946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148208947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148208966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148208966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148208966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:15:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:49,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:15:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/232c221d26ab4eb1b75bf753233ba275 2024-11-21T00:15:49,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/30b00ef46e9c4904a566892a438c0bfd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd 2024-11-21T00:15:49,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd, entries=250, sequenceid=131, filesize=47.2 K 2024-11-21T00:15:49,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/eb7ac8634bc246e38afd72e055c83ad2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2 2024-11-21T00:15:49,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T00:15:49,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/232c221d26ab4eb1b75bf753233ba275 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275 2024-11-21T00:15:49,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:15:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:49,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T00:15:49,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1076ms, sequenceid=131, compaction requested=true 2024-11-21T00:15:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:49,347 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:49,347 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:49,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:49,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:49,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:49,355 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158441 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:49,355 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:49,355 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,356 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=154.7 K 2024-11-21T00:15:49,356 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,356 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd] 2024-11-21T00:15:49,356 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:49,356 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:49,356 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,356 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/6bc7f0bb1f5e43d1911c18460701eb2b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=47.1 K 2024-11-21T00:15:49,360 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc7f0bb1f5e43d1911c18460701eb2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:49,360 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4845c8aab8ec47bbb0b3bcdcc7513fb2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:49,361 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f790ee3366b148259f8a2e7288c47fb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732148145936 2024-11-21T00:15:49,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7fbbccf11848a2b2b07617cc38c5fd, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732148145936 2024-11-21T00:15:49,361 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 7745c76b25a04823b81c4fddd3c3fa27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148146898 2024-11-21T00:15:49,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a651a7c7dc242628d956f458721547f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148146898 2024-11-21T00:15:49,364 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b00ef46e9c4904a566892a438c0bfd, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147098 2024-11-21T00:15:49,364 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7ac8634bc246e38afd72e055c83ad2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147100 2024-11-21T00:15:49,374 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:49,380 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#121 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:49,381 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5e3086cb759641d6a0e1c2c246b6e997 is 50, key is test_row_0/B:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:49,388 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210a1aa1ef1fb34cfebf31473afcd7e8ac_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:49,391 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210a1aa1ef1fb34cfebf31473afcd7e8ac_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:49,392 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210a1aa1ef1fb34cfebf31473afcd7e8ac_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:49,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741967_1143 (size=12341) 2024-11-21T00:15:49,405 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5e3086cb759641d6a0e1c2c246b6e997 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5e3086cb759641d6a0e1c2c246b6e997 2024-11-21T00:15:49,413 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into 5e3086cb759641d6a0e1c2c246b6e997(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:49,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:49,414 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=12, startTime=1732148149347; duration=0sec 2024-11-21T00:15:49,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:49,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:15:49,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:15:49,415 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:15:49,416 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:15:49,416 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,416 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a0220edf0867478f9bfeaa90fb39cf06, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=47.1 K 2024-11-21T00:15:49,416 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a0220edf0867478f9bfeaa90fb39cf06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732148145619 2024-11-21T00:15:49,416 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 065dfd09ccd3450985b606f14f6ce0d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732148145936 2024-11-21T00:15:49,417 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting dca611da13bd47cfa5a942c1f8e9785e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148146898 2024-11-21T00:15:49,417 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 232c221d26ab4eb1b75bf753233ba275, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147100 2024-11-21T00:15:49,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741968_1144 (size=4469) 2024-11-21T00:15:49,443 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#122 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:49,443 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/ac2757ec427c4f89b8ea980fe8529d2d is 50, key is test_row_0/C:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:49,445 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#120 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:49,446 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/17fed02501154ae0b1243239e9bd773c is 175, key is test_row_0/A:col10/1732148148271/Put/seqid=0 2024-11-21T00:15:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741970_1146 (size=31295) 2024-11-21T00:15:49,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:15:49,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:49,475 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-21T00:15:49,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148148309, max timestamp=1732148149473Num uniques 859; 2024-11-21T00:15:49,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:49,479 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/17fed02501154ae0b1243239e9bd773c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c 2024-11-21T00:15:49,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148148309, max timestamp=1732148149473Num uniques 859; 2024-11-21T00:15:49,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:49,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148148309, max timestamp=1732148149473Num uniques 859; 2024-11-21T00:15:49,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:15:49,492 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 17fed02501154ae0b1243239e9bd773c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:49,492 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:49,492 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=12, startTime=1732148149347; duration=0sec 2024-11-21T00:15:49,493 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:49,493 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:15:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:49,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:49,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741969_1145 (size=12341) 2024-11-21T00:15:49,520 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/ac2757ec427c4f89b8ea980fe8529d2d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ac2757ec427c4f89b8ea980fe8529d2d 2024-11-21T00:15:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d7e7267660dd4ce193b1943c6cb6a88a_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148148309/Put/seqid=0 2024-11-21T00:15:49,531 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into ac2757ec427c4f89b8ea980fe8529d2d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:49,531 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:49,532 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=12, startTime=1732148149348; duration=0sec 2024-11-21T00:15:49,532 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:49,532 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148149516, max timestamp=1732148149535Num uniques 859; 2024-11-21T00:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148149518, max timestamp=1732148149535Num uniques 859; 2024-11-21T00:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:49,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148149518, max timestamp=1732148149535Num uniques 859; 2024-11-21T00:15:49,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:15:49,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741971_1147 (size=14794) 2024-11-21T00:15:49,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732148149557, max timestamp=1732148149581Num uniques 1118; 2024-11-21T00:15:49,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=19138, suffix heap size=144240,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=418 2024-11-21T00:15:49,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732148149569, max timestamp=1732148149581Num uniques 1018; 2024-11-21T00:15:49,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T00:15:49,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148149571, max timestamp=1732148149581Num uniques 1018; 2024-11-21T00:15:49,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T00:15:49,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732148149617, max timestamp=1732148149667Num uniques 1277; 2024-11-21T00:15:49,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=58482, new segment data size=10392, suffix heap size=160776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1277, new segment cells count=227 2024-11-21T00:15:49,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732148149617, max timestamp=1732148149667Num uniques 1077; 2024-11-21T00:15:49,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-21T00:15:49,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732148149617, max timestamp=1732148149667Num uniques 1027; 2024-11-21T00:15:49,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-21T00:15:49,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=436, cellSize=19966, totalHeapSize=45584, min timestamp=1732148149696, max timestamp=1732148149748Num uniques 1086; 2024-11-21T00:15:49,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=19966, suffix heap size=140912,new segment heap size=45584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=436 2024-11-21T00:15:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732148149738, max timestamp=1732148149748Num uniques 1086; 2024-11-21T00:15:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-21T00:15:49,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732148149738, max timestamp=1732148149748Num uniques 1086; 2024-11-21T00:15:49,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-21T00:15:49,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732148149792, max timestamp=1732148149827Num uniques 1045; 2024-11-21T00:15:49,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732148149792, max timestamp=1732148149827Num uniques 1295; 2024-11-21T00:15:49,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-21T00:15:49,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=59310, new segment data size=15800, suffix heap size=162648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1295, new segment cells count=345 2024-11-21T00:15:49,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732148149792, max timestamp=1732148149827Num uniques 1045; 2024-11-21T00:15:49,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-21T00:15:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:49,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148209888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148209892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148209894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148209898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148209900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:49,954 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d7e7267660dd4ce193b1943c6cb6a88a_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7e7267660dd4ce193b1943c6cb6a88a_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1789c63c9fa04b3b8a97ee5e7c896479, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:49,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1789c63c9fa04b3b8a97ee5e7c896479 is 175, key is test_row_0/A:col10/1732148148309/Put/seqid=0 2024-11-21T00:15:49,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741972_1148 (size=39749) 2024-11-21T00:15:49,984 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1789c63c9fa04b3b8a97ee5e7c896479 2024-11-21T00:15:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/fd209b500bae427293aceecd0cb24b4e is 50, key is test_row_0/B:col10/1732148148309/Put/seqid=0 2024-11-21T00:15:50,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148210006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148210007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148210007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148210017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148210017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741973_1149 (size=12151) 2024-11-21T00:15:50,033 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/fd209b500bae427293aceecd0cb24b4e 2024-11-21T00:15:50,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/0c22d557a8724b32885ae32856d5d80b is 50, key is test_row_0/C:col10/1732148148309/Put/seqid=0 2024-11-21T00:15:50,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741974_1150 (size=12151) 2024-11-21T00:15:50,079 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/0c22d557a8724b32885ae32856d5d80b 2024-11-21T00:15:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1789c63c9fa04b3b8a97ee5e7c896479 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479 2024-11-21T00:15:50,095 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479, entries=200, sequenceid=153, filesize=38.8 K 2024-11-21T00:15:50,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/fd209b500bae427293aceecd0cb24b4e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e 2024-11-21T00:15:50,103 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T00:15:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/0c22d557a8724b32885ae32856d5d80b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b 2024-11-21T00:15:50,113 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T00:15:50,116 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=161.02 KB/164880 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 640ms, sequenceid=153, compaction requested=false 2024-11-21T00:15:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T00:15:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T00:15:50,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T00:15:50,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-21T00:15:50,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4260 sec 2024-11-21T00:15:50,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-11-21T00:15:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:15:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:15:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:15:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148210226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148210227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148210230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148210231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148210239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6bf9de9a6e54d7c8582c69d4a24af46_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:50,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741975_1151 (size=14944) 2024-11-21T00:15:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148210347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148210347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148210347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148210356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148210547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148210557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148210558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148210558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148210566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:50,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6bf9de9a6e54d7c8582c69d4a24af46_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6bf9de9a6e54d7c8582c69d4a24af46_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:50,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/33cb000f448a495eb45209ed1aae4022, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:50,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/33cb000f448a495eb45209ed1aae4022 is 175, key is test_row_0/A:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741976_1152 (size=39899) 2024-11-21T00:15:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:15:50,861 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:50,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T00:15:50,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:50,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:50,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:50,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148210866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148210866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148210866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148210887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:51,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T00:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148211056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:51,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T00:15:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,241 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/33cb000f448a495eb45209ed1aae4022 2024-11-21T00:15:51,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/badf580e5c9b4a21b81864c2f066a477 is 50, key is test_row_0/B:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741977_1153 (size=12251) 2024-11-21T00:15:51,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/badf580e5c9b4a21b81864c2f066a477 2024-11-21T00:15:51,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T00:15:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:51,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:51,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/8b37eb0778ea4014aa39e5c8e55c1092 is 50, key is test_row_0/C:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:51,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148211377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:51,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148211377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:51,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148211377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148211406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:51,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741978_1154 (size=12251) 2024-11-21T00:15:51,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/8b37eb0778ea4014aa39e5c8e55c1092 2024-11-21T00:15:51,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/33cb000f448a495eb45209ed1aae4022 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022 2024-11-21T00:15:51,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022, entries=200, sequenceid=262, filesize=39.0 K 2024-11-21T00:15:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/badf580e5c9b4a21b81864c2f066a477 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477 2024-11-21T00:15:51,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477, entries=150, sequenceid=262, filesize=12.0 K 2024-11-21T00:15:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/8b37eb0778ea4014aa39e5c8e55c1092 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092 2024-11-21T00:15:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092, entries=150, sequenceid=262, filesize=12.0 K 2024-11-21T00:15:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~467.81 KB/479040, currentSize=20.13 KB/20610 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1274ms, sequenceid=262, compaction requested=true 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:51,498 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:51,498 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:51,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T00:15:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:51,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,498 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-21T00:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,501 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:51,501 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:51,501 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,501 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=108.3 K 2024-11-21T00:15:51,501 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,501 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022] 2024-11-21T00:15:51,502 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:51,502 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:51,502 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:51,502 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5e3086cb759641d6a0e1c2c246b6e997, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=35.9 K 2024-11-21T00:15:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,504 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e3086cb759641d6a0e1c2c246b6e997, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147100 2024-11-21T00:15:51,505 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17fed02501154ae0b1243239e9bd773c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147100 2024-11-21T00:15:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,505 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fd209b500bae427293aceecd0cb24b4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732148148309 2024-11-21T00:15:51,505 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1789c63c9fa04b3b8a97ee5e7c896479, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732148148309 2024-11-21T00:15:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,506 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting badf580e5c9b4a21b81864c2f066a477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,507 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33cb000f448a495eb45209ed1aae4022, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2cb71f4cb52430bb6f4836ac3d1c658_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_1/A:col10/1732148150237/Put/seqid=0 2024-11-21T00:15:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,535 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:51,535 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5b34800692e44e498661028ab32dc59c is 50, key is test_row_0/B:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,555 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741980_1156 (size=12593) 2024-11-21T00:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741979_1155 (size=9914) 2024-11-21T00:15:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,589 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112138b6acfd85024389be214fcdd202951b_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,594 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112138b6acfd85024389be214fcdd202951b_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:51,594 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112138b6acfd85024389be214fcdd202951b_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,596 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2cb71f4cb52430bb6f4836ac3d1c658_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2cb71f4cb52430bb6f4836ac3d1c658_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6d89992a40e648e1a02da03ec611b311, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6d89992a40e648e1a02da03ec611b311 is 175, key is test_row_1/A:col10/1732148150237/Put/seqid=0 2024-11-21T00:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741981_1157 (size=4469) 2024-11-21T00:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,639 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#131 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,640 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3b311ebac59c4cd9919e49ff93e42170 is 175, key is test_row_0/A:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741982_1158 (size=22561) 2024-11-21T00:15:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,673 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6d89992a40e648e1a02da03ec611b311 2024-11-21T00:15:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741983_1159 (size=31547) 2024-11-21T00:15:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/70ba6d65f48e4748b6bd23ef9969ff4c is 50, key is test_row_1/B:col10/1732148150237/Put/seqid=0 2024-11-21T00:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741984_1160 (size=9857) 2024-11-21T00:15:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,737 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/70ba6d65f48e4748b6bd23ef9969ff4c 2024-11-21T00:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f13806efb67145ef83e6cf0a07da5e07 is 50, key is test_row_1/C:col10/1732148150237/Put/seqid=0 2024-11-21T00:15:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741985_1161 (size=9857) 2024-11-21T00:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:51,999 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5b34800692e44e498661028ab32dc59c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5b34800692e44e498661028ab32dc59c 2024-11-21T00:15:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,014 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into 5b34800692e44e498661028ab32dc59c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:52,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:52,014 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148151498; duration=0sec 2024-11-21T00:15:52,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:52,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:15:52,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,016 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:52,016 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:15:52,016 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:52,016 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ac2757ec427c4f89b8ea980fe8529d2d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=35.9 K 2024-11-21T00:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,018 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2757ec427c4f89b8ea980fe8529d2d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148147100 2024-11-21T00:15:52,018 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c22d557a8724b32885ae32856d5d80b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732148148309 2024-11-21T00:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,019 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b37eb0778ea4014aa39e5c8e55c1092, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,041 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,041 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a5feb0644d8942c5894dfae2ee798ac9 is 50, key is test_row_0/C:col10/1732148150221/Put/seqid=0 2024-11-21T00:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741986_1162 (size=12593) 2024-11-21T00:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,080 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a5feb0644d8942c5894dfae2ee798ac9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a5feb0644d8942c5894dfae2ee798ac9 2024-11-21T00:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,085 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3b311ebac59c4cd9919e49ff93e42170 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170 2024-11-21T00:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,103 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into a5feb0644d8942c5894dfae2ee798ac9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:52,103 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,104 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148151498; duration=0sec 2024-11-21T00:15:52,104 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:52,104 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,106 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 3b311ebac59c4cd9919e49ff93e42170(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:52,106 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:52,106 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148151498; duration=0sec 2024-11-21T00:15:52,106 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:52,106 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:52,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,181 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f13806efb67145ef83e6cf0a07da5e07 2024-11-21T00:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/6d89992a40e648e1a02da03ec611b311 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311 2024-11-21T00:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,198 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311, entries=100, sequenceid=268, filesize=22.0 K 2024-11-21T00:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/70ba6d65f48e4748b6bd23ef9969ff4c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c 2024-11-21T00:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,210 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c, entries=100, sequenceid=268, filesize=9.6 K 2024-11-21T00:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/f13806efb67145ef83e6cf0a07da5e07 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07 2024-11-21T00:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,224 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07, entries=100, sequenceid=268, filesize=9.6 K 2024-11-21T00:15:52,229 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=80.51 KB/82440 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 730ms, sequenceid=268, compaction requested=false 2024-11-21T00:15:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T00:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T00:15:52,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T00:15:52,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3670 sec 2024-11-21T00:15:52,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3740 sec 2024-11-21T00:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:52,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:52,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210e07c2adf25e420c847c441b4a093fb4_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:52,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741987_1163 (size=14994) 2024-11-21T00:15:52,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:52,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210e07c2adf25e420c847c441b4a093fb4_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210e07c2adf25e420c847c441b4a093fb4_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:52,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/c4e9ff702ba14867807acc11f464e001, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:52,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/c4e9ff702ba14867807acc11f464e001 is 175, key is test_row_0/A:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:52,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148212316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741988_1164 (size=39949) 2024-11-21T00:15:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148212387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,392 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148212397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,407 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148212397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,410 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148212427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148212429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,432 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148212636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:52,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/c4e9ff702ba14867807acc11f464e001 2024-11-21T00:15:52,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a69a776309942d3a469633ecde7ea35 is 50, key is test_row_0/B:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:52,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741989_1165 (size=12301) 2024-11-21T00:15:52,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a69a776309942d3a469633ecde7ea35 2024-11-21T00:15:52,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/ae1544a16b8f4ccd99348580d137294d is 50, key is test_row_0/C:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:52,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741990_1166 (size=12301) 2024-11-21T00:15:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148212946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:15:53,008 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:53,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T00:15:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:15:53,028 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:53,029 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:53,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:15:53,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:15:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:53,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/ae1544a16b8f4ccd99348580d137294d 2024-11-21T00:15:53,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/c4e9ff702ba14867807acc11f464e001 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001 2024-11-21T00:15:53,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001, entries=200, sequenceid=288, filesize=39.0 K 2024-11-21T00:15:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3a69a776309942d3a469633ecde7ea35 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35 2024-11-21T00:15:53,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T00:15:53,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/ae1544a16b8f4ccd99348580d137294d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d 2024-11-21T00:15:53,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T00:15:53,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1014ms, sequenceid=288, compaction requested=true 2024-11-21T00:15:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:53,252 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:53,252 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:53,253 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:53,253 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:53,254 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,254 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5b34800692e44e498661028ab32dc59c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=33.9 K 2024-11-21T00:15:53,256 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:53,256 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b34800692e44e498661028ab32dc59c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:53,256 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:53,256 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,256 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=91.9 K 2024-11-21T00:15:53,256 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ba6d65f48e4748b6bd23ef9969ff4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732148150229 2024-11-21T00:15:53,256 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,256 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001] 2024-11-21T00:15:53,257 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a69a776309942d3a469633ecde7ea35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:53,257 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b311ebac59c4cd9919e49ff93e42170, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:53,265 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d89992a40e648e1a02da03ec611b311, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732148150229 2024-11-21T00:15:53,267 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4e9ff702ba14867807acc11f464e001, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:53,294 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:53,295 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/56f655c2a97349fdb274d5d8cf332a01 is 50, key is test_row_0/B:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:53,310 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:53,318 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121bfc471c8b8d642bb85c88c2d14ae730d_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:53,321 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121bfc471c8b8d642bb85c88c2d14ae730d_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:53,321 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bfc471c8b8d642bb85c88c2d14ae730d_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:15:53,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741991_1167 (size=12745) 2024-11-21T00:15:53,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:15:53,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:53,368 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/56f655c2a97349fdb274d5d8cf332a01 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/56f655c2a97349fdb274d5d8cf332a01 2024-11-21T00:15:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741992_1168 (size=4469) 2024-11-21T00:15:53,405 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into 56f655c2a97349fdb274d5d8cf332a01(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:53,406 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:53,406 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148153252; duration=0sec 2024-11-21T00:15:53,406 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:53,406 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:15:53,406 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:53,408 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:53,408 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:15:53,408 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:53,408 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a5feb0644d8942c5894dfae2ee798ac9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=33.9 K 2024-11-21T00:15:53,413 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a5feb0644d8942c5894dfae2ee798ac9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732148149885 2024-11-21T00:15:53,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f13806efb67145ef83e6cf0a07da5e07, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732148150229 2024-11-21T00:15:53,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T00:15:53,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T00:15:53,417 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1544a16b8f4ccd99348580d137294d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112100be0fa88f914f7c8acf8e6afb839566_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148152295/Put/seqid=0 2024-11-21T00:15:53,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741993_1169 (size=12454) 2024-11-21T00:15:53,466 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#141 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:53,466 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/d6c82af463a448438fa6d3e651925db9 is 50, key is test_row_0/C:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:53,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:53,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741994_1170 (size=12745) 2024-11-21T00:15:53,524 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/d6c82af463a448438fa6d3e651925db9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/d6c82af463a448438fa6d3e651925db9 2024-11-21T00:15:53,539 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into d6c82af463a448438fa6d3e651925db9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:53,539 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:53,539 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148153253; duration=0sec 2024-11-21T00:15:53,540 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:53,540 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:15:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148213624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:15:53,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148213736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:53,794 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:53,795 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ece6c9f504d848fdcfd3b8ee7b75f is 175, key is test_row_0/A:col10/1732148152227/Put/seqid=0 2024-11-21T00:15:53,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741995_1171 (size=31699) 2024-11-21T00:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:53,872 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112100be0fa88f914f7c8acf8e6afb839566_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112100be0fa88f914f7c8acf8e6afb839566_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:53,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/2c89407190144abdb783fdb5ff2669c1, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:53,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/2c89407190144abdb783fdb5ff2669c1 is 175, key is test_row_0/A:col10/1732148152295/Put/seqid=0 2024-11-21T00:15:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741996_1172 (size=31255) 2024-11-21T00:15:53,902 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/2c89407190144abdb783fdb5ff2669c1 2024-11-21T00:15:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/0089d10852064a13a694f5715aa21a61 is 50, key is test_row_0/B:col10/1732148152295/Put/seqid=0 2024-11-21T00:15:53,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148213947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:53,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741997_1173 (size=12301) 2024-11-21T00:15:53,951 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/0089d10852064a13a694f5715aa21a61 2024-11-21T00:15:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/97d64e8ba76947d4b7f86a630dac94c1 is 50, key is test_row_0/C:col10/1732148152295/Put/seqid=0 2024-11-21T00:15:53,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741998_1174 (size=12301) 2024-11-21T00:15:53,984 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/97d64e8ba76947d4b7f86a630dac94c1 2024-11-21T00:15:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/2c89407190144abdb783fdb5ff2669c1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1 2024-11-21T00:15:54,014 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1, entries=150, sequenceid=303, filesize=30.5 K 2024-11-21T00:15:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/0089d10852064a13a694f5715aa21a61 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61 2024-11-21T00:15:54,025 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T00:15:54,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/97d64e8ba76947d4b7f86a630dac94c1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1 2024-11-21T00:15:54,050 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T00:15:54,051 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 707ms, sequenceid=303, compaction requested=false 2024-11-21T00:15:54,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:54,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T00:15:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T00:15:54,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T00:15:54,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0240 sec 2024-11-21T00:15:54,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0510 sec 2024-11-21T00:15:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:15:54,147 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:54,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T00:15:54,153 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:54,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:54,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:54,221 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ece6c9f504d848fdcfd3b8ee7b75f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f 2024-11-21T00:15:54,229 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 3a6ece6c9f504d848fdcfd3b8ee7b75f(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:54,229 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:54,230 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148153252; duration=0sec 2024-11-21T00:15:54,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:54,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:15:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:54,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:54,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c1ee51c2a8e24026b102e5549f8bdd8e_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:54,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741999_1175 (size=14994) 2024-11-21T00:15:54,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:54,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:54,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148214309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148214396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,401 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148214416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,418 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148214426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148214426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,431 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148214446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,452 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:54,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148214636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:54,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:54,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c1ee51c2a8e24026b102e5549f8bdd8e_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1ee51c2a8e24026b102e5549f8bdd8e_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:54,700 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/38e08f0245464e76813ede7a0e16e5c4, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:54,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/38e08f0245464e76813ede7a0e16e5c4 is 175, key is test_row_0/A:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742000_1176 (size=39949) 2024-11-21T00:15:54,721 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/38e08f0245464e76813ede7a0e16e5c4 2024-11-21T00:15:54,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9e09e0b775fb47b88ce0a856f82caf5e is 50, key is test_row_0/B:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:54,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742001_1177 (size=12301) 2024-11-21T00:15:54,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:54,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:54,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:54,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148214949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:55,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9e09e0b775fb47b88ce0a856f82caf5e 2024-11-21T00:15:55,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/037218ceb9af4932b65ab899a340c295 is 50, key is test_row_0/C:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742002_1178 (size=12301) 2024-11-21T00:15:55,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/037218ceb9af4932b65ab899a340c295 2024-11-21T00:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/38e08f0245464e76813ede7a0e16e5c4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4 2024-11-21T00:15:55,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4, entries=200, sequenceid=324, filesize=39.0 K 2024-11-21T00:15:55,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9e09e0b775fb47b88ce0a856f82caf5e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e 2024-11-21T00:15:55,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-21T00:15:55,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/037218ceb9af4932b65ab899a340c295 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295 2024-11-21T00:15:55,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295, entries=150, sequenceid=324, filesize=12.0 K 2024-11-21T00:15:55,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:55,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 998ms, sequenceid=324, compaction requested=true 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:55,257 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:15:55,259 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:55,259 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:55,260 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:55,260 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,260 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=100.5 K 2024-11-21T00:15:55,260 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,260 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:55,260 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4] 2024-11-21T00:15:55,260 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:55,260 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,260 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/56f655c2a97349fdb274d5d8cf332a01, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.5 K 2024-11-21T00:15:55,261 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f655c2a97349fdb274d5d8cf332a01, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:55,261 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6ece6c9f504d848fdcfd3b8ee7b75f, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:55,261 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0089d10852064a13a694f5715aa21a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148152295 2024-11-21T00:15:55,261 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c89407190144abdb783fdb5ff2669c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148152295 2024-11-21T00:15:55,262 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e08f0245464e76813ede7a0e16e5c4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:15:55,262 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e09e0b775fb47b88ce0a856f82caf5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:15:55,277 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:55,281 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112142240e2ffa5043e68a3c552e8fd5e287_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:55,283 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112142240e2ffa5043e68a3c552e8fd5e287_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:55,284 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112142240e2ffa5043e68a3c552e8fd5e287_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:55,284 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:55,284 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/ca75c65a7ca54360b56918ce74b53168 is 50, key is test_row_0/B:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:55,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742003_1179 (size=12847) 2024-11-21T00:15:55,325 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/ca75c65a7ca54360b56918ce74b53168 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ca75c65a7ca54360b56918ce74b53168 2024-11-21T00:15:55,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742004_1180 (size=4469) 2024-11-21T00:15:55,343 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#147 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:55,343 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/fa5e2b80c64c448da9e028d41df03e75 is 175, key is test_row_0/A:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:55,348 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into ca75c65a7ca54360b56918ce74b53168(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:55,348 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:55,349 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148155257; duration=0sec 2024-11-21T00:15:55,349 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:55,349 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:15:55,349 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:55,350 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:55,350 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:15:55,350 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,350 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/d6c82af463a448438fa6d3e651925db9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.5 K 2024-11-21T00:15:55,351 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c82af463a448438fa6d3e651925db9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148152160 2024-11-21T00:15:55,352 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d64e8ba76947d4b7f86a630dac94c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148152295 2024-11-21T00:15:55,354 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 037218ceb9af4932b65ab899a340c295, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:15:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742005_1181 (size=31801) 2024-11-21T00:15:55,378 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:55,378 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/b6956131de924297a4ecb74b20fbce4b is 50, key is test_row_0/C:col10/1732148154258/Put/seqid=0 2024-11-21T00:15:55,387 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/fa5e2b80c64c448da9e028d41df03e75 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75 2024-11-21T00:15:55,396 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into fa5e2b80c64c448da9e028d41df03e75(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:55,396 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:55,396 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148155257; duration=0sec 2024-11-21T00:15:55,396 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:55,396 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:15:55,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742006_1182 (size=12847) 2024-11-21T00:15:55,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:15:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:55,409 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:55,418 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/b6956131de924297a4ecb74b20fbce4b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b6956131de924297a4ecb74b20fbce4b 2024-11-21T00:15:55,427 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into b6956131de924297a4ecb74b20fbce4b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:15:55,427 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:55,427 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148155257; duration=0sec 2024-11-21T00:15:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126065b033bf64faba1ae1c56c30e104d_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148154299/Put/seqid=0 2024-11-21T00:15:55,428 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:55,428 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:15:55,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742007_1183 (size=12454) 2024-11-21T00:15:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148215589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148215696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:55,873 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126065b033bf64faba1ae1c56c30e104d_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126065b033bf64faba1ae1c56c30e104d_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b6d6f48f65a040c184eecfaf9ade02c3, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:55,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b6d6f48f65a040c184eecfaf9ade02c3 is 175, key is test_row_0/A:col10/1732148154299/Put/seqid=0 2024-11-21T00:15:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742008_1184 (size=31255) 2024-11-21T00:15:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148215907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148216233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:56,292 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b6d6f48f65a040c184eecfaf9ade02c3 2024-11-21T00:15:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/03c8cb4a6f7547ed8e64764271ba29c2 is 50, key is test_row_0/B:col10/1732148154299/Put/seqid=0 2024-11-21T00:15:56,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742009_1185 (size=12301) 2024-11-21T00:15:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:56,705 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/03c8cb4a6f7547ed8e64764271ba29c2 2024-11-21T00:15:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a692f1c681244f5c90dcf9ab679021b1 is 50, key is test_row_0/C:col10/1732148154299/Put/seqid=0 2024-11-21T00:15:56,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742010_1186 (size=12301) 2024-11-21T00:15:56,737 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a692f1c681244f5c90dcf9ab679021b1 2024-11-21T00:15:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b6d6f48f65a040c184eecfaf9ade02c3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3 2024-11-21T00:15:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148216748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:56,756 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3, entries=150, sequenceid=340, filesize=30.5 K 2024-11-21T00:15:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/03c8cb4a6f7547ed8e64764271ba29c2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2 2024-11-21T00:15:56,768 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T00:15:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/a692f1c681244f5c90dcf9ab679021b1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1 2024-11-21T00:15:56,783 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T00:15:56,787 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1379ms, sequenceid=340, compaction requested=false 2024-11-21T00:15:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T00:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T00:15:56,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T00:15:56,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6370 sec 2024-11-21T00:15:56,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.6500 sec 2024-11-21T00:15:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:57,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:57,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121af041cb24a12414392a0782445ee58a3_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:57,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742011_1187 (size=14994) 2024-11-21T00:15:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148217828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:57,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148217937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148218146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:58,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121af041cb24a12414392a0782445ee58a3_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af041cb24a12414392a0782445ee58a3_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:58,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/ad76c21f03a948ca82916b7a924993c5, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:58,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/ad76c21f03a948ca82916b7a924993c5 is 175, key is test_row_0/A:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742012_1188 (size=39949) 2024-11-21T00:15:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:15:58,337 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:15:58,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:15:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T00:15:58,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:15:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:15:58,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:15:58,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:15:58,348 INFO [master/5ed4808ef0e6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T00:15:58,348 INFO [master/5ed4808ef0e6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T00:15:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59024 deadline: 1732148218416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,418 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:15:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59042 deadline: 1732148218456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148218456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58994 deadline: 1732148218456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1732148218467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,468 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,468 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,468 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in cb31c4d3052e5d7cdaabfc9b52a02c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:58,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/ad76c21f03a948ca82916b7a924993c5 2024-11-21T00:15:58,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:15:58,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3cf9b8abf4704f4d8ef1f53b82076538 is 50, key is test_row_0/B:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:58,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742013_1189 (size=12301) 2024-11-21T00:15:58,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:58,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:58,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:58,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:15:58,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:15:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148218966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:15:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:15:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:15:59,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3cf9b8abf4704f4d8ef1f53b82076538 2024-11-21T00:15:59,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/5968680864474fdabf7b8893720e2803 is 50, key is test_row_0/C:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:59,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742014_1190 (size=12301) 2024-11-21T00:15:59,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:59,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:59,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:15:59,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/5968680864474fdabf7b8893720e2803 2024-11-21T00:15:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/ad76c21f03a948ca82916b7a924993c5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5 2024-11-21T00:15:59,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5, entries=200, sequenceid=360, filesize=39.0 K 2024-11-21T00:15:59,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3cf9b8abf4704f4d8ef1f53b82076538 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538 2024-11-21T00:15:59,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538, entries=150, sequenceid=360, filesize=12.0 K 2024-11-21T00:15:59,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/5968680864474fdabf7b8893720e2803 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803 2024-11-21T00:15:59,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803, entries=150, sequenceid=360, filesize=12.0 K 2024-11-21T00:15:59,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1757ms, sequenceid=360, compaction requested=true 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:59,525 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:15:59,525 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:15:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:15:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:15:59,527 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:59,527 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:15:59,527 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:15:59,527 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:15:59,527 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,527 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,527 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ca75c65a7ca54360b56918ce74b53168, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.6 K 2024-11-21T00:15:59,527 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=100.6 K 2024-11-21T00:15:59,527 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,527 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5] 2024-11-21T00:15:59,528 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ca75c65a7ca54360b56918ce74b53168, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:15:59,528 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5e2b80c64c448da9e028d41df03e75, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:15:59,528 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c8cb4a6f7547ed8e64764271ba29c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148154299 2024-11-21T00:15:59,528 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d6f48f65a040c184eecfaf9ade02c3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148154299 2024-11-21T00:15:59,528 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf9b8abf4704f4d8ef1f53b82076538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:15:59,529 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad76c21f03a948ca82916b7a924993c5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:15:59,538 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:59,542 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:59,543 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/b0e32023111e4518b145da888095fabe is 50, key is test_row_0/B:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:59,545 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217c781e0a7c224db89953599668213a9c_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217c781e0a7c224db89953599668213a9c_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c781e0a7c224db89953599668213a9c_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:59,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:15:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:15:59,572 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:15:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:15:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742016_1192 (size=4469) 2024-11-21T00:15:59,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742015_1191 (size=12949) 2024-11-21T00:15:59,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121966067e2127446aaa101aee032c79b95_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148157821/Put/seqid=0 2024-11-21T00:15:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742017_1193 (size=12454) 2024-11-21T00:15:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:15:59,631 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121966067e2127446aaa101aee032c79b95_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121966067e2127446aaa101aee032c79b95_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:59,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/352c75fb4028428eadbde001e6b28be8, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:15:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/352c75fb4028428eadbde001e6b28be8 is 175, key is test_row_0/A:col10/1732148157821/Put/seqid=0 2024-11-21T00:15:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742018_1194 (size=31255) 2024-11-21T00:15:59,645 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/352c75fb4028428eadbde001e6b28be8 2024-11-21T00:15:59,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3475c451315b4ad886016e9d0d863321 is 50, key is test_row_0/B:col10/1732148157821/Put/seqid=0 2024-11-21T00:15:59,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742019_1195 (size=12301) 2024-11-21T00:15:59,680 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3475c451315b4ad886016e9d0d863321 2024-11-21T00:15:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/868b9b4a16394af39fdecbe63018e075 is 50, key is test_row_0/C:col10/1732148157821/Put/seqid=0 2024-11-21T00:15:59,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742020_1196 (size=12301) 2024-11-21T00:15:59,981 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:15:59,981 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b1a16afe4cab4b85b64e34780a62a5ef is 175, key is test_row_0/A:col10/1732148155587/Put/seqid=0 2024-11-21T00:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:15:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:15:59,997 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/b0e32023111e4518b145da888095fabe as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/b0e32023111e4518b145da888095fabe 2024-11-21T00:16:00,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742021_1197 (size=31903) 2024-11-21T00:16:00,006 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into b0e32023111e4518b145da888095fabe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:00,006 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:00,007 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148159525; duration=0sec 2024-11-21T00:16:00,007 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:00,007 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:16:00,007 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:00,013 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:00,013 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:16:00,013 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,013 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b6956131de924297a4ecb74b20fbce4b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.6 K 2024-11-21T00:16:00,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b6956131de924297a4ecb74b20fbce4b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732148153582 2024-11-21T00:16:00,014 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a692f1c681244f5c90dcf9ab679021b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148154299 2024-11-21T00:16:00,016 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/b1a16afe4cab4b85b64e34780a62a5ef as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef 2024-11-21T00:16:00,017 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5968680864474fdabf7b8893720e2803, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:16:00,063 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:00,064 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/c8fd1c03bacf475aba956b0339c1adfd is 50, key is test_row_0/C:col10/1732148155587/Put/seqid=0 2024-11-21T00:16:00,065 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into b1a16afe4cab4b85b64e34780a62a5ef(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:00,065 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:00,065 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148159525; duration=0sec 2024-11-21T00:16:00,066 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:00,066 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:16:00,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742022_1198 (size=12949) 2024-11-21T00:16:00,122 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/868b9b4a16394af39fdecbe63018e075 2024-11-21T00:16:00,134 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/c8fd1c03bacf475aba956b0339c1adfd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c8fd1c03bacf475aba956b0339c1adfd 2024-11-21T00:16:00,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/352c75fb4028428eadbde001e6b28be8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8 2024-11-21T00:16:00,155 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8, entries=150, sequenceid=373, filesize=30.5 K 2024-11-21T00:16:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/3475c451315b4ad886016e9d0d863321 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321 2024-11-21T00:16:00,156 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into c8fd1c03bacf475aba956b0339c1adfd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:00,157 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:00,157 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148159525; duration=0sec 2024-11-21T00:16:00,157 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:00,157 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:16:00,162 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321, entries=150, sequenceid=373, filesize=12.0 K 2024-11-21T00:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/868b9b4a16394af39fdecbe63018e075 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075 2024-11-21T00:16:00,176 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075, entries=150, sequenceid=373, filesize=12.0 K 2024-11-21T00:16:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 606ms, sequenceid=373, compaction requested=false 2024-11-21T00:16:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T00:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T00:16:00,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T00:16:00,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-11-21T00:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:00,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T00:16:00,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:00,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:00,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8470 sec 2024-11-21T00:16:00,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212a0bb210484546f48b5db9e9243b8026_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148220243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:00,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742023_1199 (size=14994) 2024-11-21T00:16:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148220356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:16:00,486 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:00,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T00:16:00,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:00,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:00,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148220566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:00,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,668 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:00,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212a0bb210484546f48b5db9e9243b8026_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a0bb210484546f48b5db9e9243b8026_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:00,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/40fcb0c93ed2419f993269239e22ef12, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:00,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/40fcb0c93ed2419f993269239e22ef12 is 175, key is test_row_0/A:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742024_1200 (size=39949) 2024-11-21T00:16:00,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:00,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148220877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:00,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:00,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:01,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/40fcb0c93ed2419f993269239e22ef12 2024-11-21T00:16:01,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9c4071323e85486386a7df1d38af74b4 is 50, key is test_row_0/B:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:01,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742025_1201 (size=12301) 2024-11-21T00:16:01,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9c4071323e85486386a7df1d38af74b4 2024-11-21T00:16:01,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/34eb4d70d162491180aff5dc81dc02d2 is 50, key is test_row_0/C:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742026_1202 (size=12301) 2024-11-21T00:16:01,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148221396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:01,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:01,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/34eb4d70d162491180aff5dc81dc02d2 2024-11-21T00:16:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/40fcb0c93ed2419f993269239e22ef12 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12 2024-11-21T00:16:01,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12, entries=200, sequenceid=396, filesize=39.0 K 2024-11-21T00:16:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/9c4071323e85486386a7df1d38af74b4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4 2024-11-21T00:16:01,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4, entries=150, sequenceid=396, filesize=12.0 K 2024-11-21T00:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/34eb4d70d162491180aff5dc81dc02d2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2 2024-11-21T00:16:01,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2, entries=150, sequenceid=396, filesize=12.0 K 2024-11-21T00:16:01,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1457ms, sequenceid=396, compaction requested=true 2024-11-21T00:16:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:01,642 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:01,642 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:01,645 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:01,645 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:01,645 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:16:01,645 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:16:01,645 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,645 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,645 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/b0e32023111e4518b145da888095fabe, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.7 K 2024-11-21T00:16:01,645 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=100.7 K 2024-11-21T00:16:01,645 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,645 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12] 2024-11-21T00:16:01,646 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1a16afe4cab4b85b64e34780a62a5ef, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:16:01,646 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e32023111e4518b145da888095fabe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:16:01,646 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3475c451315b4ad886016e9d0d863321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148157821 2024-11-21T00:16:01,646 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352c75fb4028428eadbde001e6b28be8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148157821 2024-11-21T00:16:01,646 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4071323e85486386a7df1d38af74b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:01,647 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40fcb0c93ed2419f993269239e22ef12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:01,657 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:01,666 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:01,666 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/d32b87c01c424ff8abb0c8757aba7c65 is 50, key is test_row_0/B:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:01,669 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f1d8593d614344738fbe3cc18563fb02_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:01,671 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f1d8593d614344738fbe3cc18563fb02_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:01,671 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f1d8593d614344738fbe3cc18563fb02_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:01,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742028_1204 (size=4469) 2024-11-21T00:16:01,704 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#165 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:01,705 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/18bb9aa27a53481096b3dc6143d3442f is 175, key is test_row_0/A:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:01,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742027_1203 (size=13051) 2024-11-21T00:16:01,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742029_1205 (size=32005) 2024-11-21T00:16:01,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T00:16:01,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,735 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:01,738 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/18bb9aa27a53481096b3dc6143d3442f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f 2024-11-21T00:16:01,750 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 18bb9aa27a53481096b3dc6143d3442f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:01,750 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:01,750 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148161642; duration=0sec 2024-11-21T00:16:01,750 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:01,750 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:16:01,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150a89711904643f8bbdf2f502c7f2413_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148160190/Put/seqid=0 2024-11-21T00:16:01,755 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:01,755 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:16:01,755 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:01,755 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c8fd1c03bacf475aba956b0339c1adfd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.7 K 2024-11-21T00:16:01,756 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8fd1c03bacf475aba956b0339c1adfd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732148155565 2024-11-21T00:16:01,760 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 868b9b4a16394af39fdecbe63018e075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148157821 2024-11-21T00:16:01,761 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34eb4d70d162491180aff5dc81dc02d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:01,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742030_1206 (size=12454) 2024-11-21T00:16:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:01,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150a89711904643f8bbdf2f502c7f2413_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150a89711904643f8bbdf2f502c7f2413_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/751756e49ef54dd9a4904491e6c21f03, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/751756e49ef54dd9a4904491e6c21f03 is 175, key is test_row_0/A:col10/1732148160190/Put/seqid=0 2024-11-21T00:16:01,779 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:01,779 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/74b709a99923400a9a4ff69566431160 is 50, key is test_row_0/C:col10/1732148160181/Put/seqid=0 2024-11-21T00:16:01,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742031_1207 (size=31255) 2024-11-21T00:16:01,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742032_1208 (size=13051) 2024-11-21T00:16:02,161 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/d32b87c01c424ff8abb0c8757aba7c65 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/d32b87c01c424ff8abb0c8757aba7c65 2024-11-21T00:16:02,196 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into d32b87c01c424ff8abb0c8757aba7c65(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:02,196 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:02,196 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148161642; duration=0sec 2024-11-21T00:16:02,196 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:02,196 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:16:02,217 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/751756e49ef54dd9a4904491e6c21f03 2024-11-21T00:16:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/cf2dfb99418942adbe0a04e523e99483 is 50, key is test_row_0/B:col10/1732148160190/Put/seqid=0 2024-11-21T00:16:02,251 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/74b709a99923400a9a4ff69566431160 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/74b709a99923400a9a4ff69566431160 2024-11-21T00:16:02,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742033_1209 (size=12301) 2024-11-21T00:16:02,272 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/cf2dfb99418942adbe0a04e523e99483 2024-11-21T00:16:02,280 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into 74b709a99923400a9a4ff69566431160(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:02,281 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:02,281 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148161642; duration=0sec 2024-11-21T00:16:02,281 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:02,281 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:16:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/6e2fdc5aef684a708df4b83e7b0cc348 is 50, key is test_row_0/C:col10/1732148160190/Put/seqid=0 2024-11-21T00:16:02,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742034_1210 (size=12301) 2024-11-21T00:16:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:02,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. as already flushing 2024-11-21T00:16:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148222533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:02,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:02,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148222646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:02,742 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/6e2fdc5aef684a708df4b83e7b0cc348 2024-11-21T00:16:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/751756e49ef54dd9a4904491e6c21f03 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03 2024-11-21T00:16:02,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03, entries=150, sequenceid=408, filesize=30.5 K 2024-11-21T00:16:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/cf2dfb99418942adbe0a04e523e99483 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483 2024-11-21T00:16:02,767 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:16:02,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/6e2fdc5aef684a708df4b83e7b0cc348 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348 2024-11-21T00:16:02,783 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:16:02,786 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1052ms, sequenceid=408, compaction requested=false 2024-11-21T00:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T00:16:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T00:16:02,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T00:16:02,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2980 sec 2024-11-21T00:16:02,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.3030 sec 2024-11-21T00:16:02,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148162483, max timestamp=1732148162859Num uniques 859; 2024-11-21T00:16:02,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:02,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148162483, max timestamp=1732148162859Num uniques 859; 2024-11-21T00:16:02,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:02,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:02,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112160ad5f14aac14f7ca79884985c20e26e_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:02,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742035_1211 (size=14994) 2024-11-21T00:16:02,891 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:02,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112160ad5f14aac14f7ca79884985c20e26e_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112160ad5f14aac14f7ca79884985c20e26e_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:02,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/946b5ce4a1fd47c08663e267eb7b8e12, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:02,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/946b5ce4a1fd47c08663e267eb7b8e12 is 175, key is test_row_0/A:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742036_1212 (size=39949) 2024-11-21T00:16:02,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=432, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/946b5ce4a1fd47c08663e267eb7b8e12 2024-11-21T00:16:02,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7a1a56995b8944c796374eadb20ec7fc is 50, key is test_row_0/B:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:02,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742037_1213 (size=12301) 2024-11-21T00:16:03,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148162932, max timestamp=1732148162996Num uniques 859; 2024-11-21T00:16:03,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:03,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148162932, max timestamp=1732148162996Num uniques 859; 2024-11-21T00:16:03,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148162932, max timestamp=1732148162996Num uniques 859; 2024-11-21T00:16:03,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:03,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148223092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:03,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148223206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:03,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:03,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7a1a56995b8944c796374eadb20ec7fc 2024-11-21T00:16:03,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/78484466cef94d3fbfb61a1872a62ae6 is 50, key is test_row_0/C:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:03,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148223420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:03,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742038_1214 (size=12301) 2024-11-21T00:16:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148223736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:03,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/78484466cef94d3fbfb61a1872a62ae6 2024-11-21T00:16:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/946b5ce4a1fd47c08663e267eb7b8e12 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12 2024-11-21T00:16:03,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12, entries=200, sequenceid=432, filesize=39.0 K 2024-11-21T00:16:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/7a1a56995b8944c796374eadb20ec7fc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc 2024-11-21T00:16:03,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc, entries=150, sequenceid=432, filesize=12.0 K 2024-11-21T00:16:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/78484466cef94d3fbfb61a1872a62ae6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6 2024-11-21T00:16:03,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6, entries=150, sequenceid=432, filesize=12.0 K 2024-11-21T00:16:03,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=123.00 KB/125950 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1009ms, sequenceid=432, compaction requested=true 2024-11-21T00:16:03,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:03,873 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:03,873 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:03,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:03,874 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:03,874 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:03,874 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:16:03,874 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:16:03,874 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:03,874 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:03,875 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/d32b87c01c424ff8abb0c8757aba7c65, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.8 K 2024-11-21T00:16:03,875 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=100.8 K 2024-11-21T00:16:03,875 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:03,875 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12] 2024-11-21T00:16:03,875 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d32b87c01c424ff8abb0c8757aba7c65, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:03,876 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bb9aa27a53481096b3dc6143d3442f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:03,876 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2dfb99418942adbe0a04e523e99483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148160190 2024-11-21T00:16:03,876 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751756e49ef54dd9a4904491e6c21f03, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148160190 2024-11-21T00:16:03,876 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a1a56995b8944c796374eadb20ec7fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:03,876 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946b5ce4a1fd47c08663e267eb7b8e12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:03,892 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:03,893 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#174 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:03,893 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/f0c7d0ed09fc40bd8abf5c60ed305550 is 50, key is test_row_0/B:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:03,897 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f600e3c4ab124dd7a5831915f7de668f_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:03,899 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f600e3c4ab124dd7a5831915f7de668f_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:03,900 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f600e3c4ab124dd7a5831915f7de668f_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:03,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742039_1215 (size=13153) 2024-11-21T00:16:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742040_1216 (size=4469) 2024-11-21T00:16:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:04,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:04,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:04,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd834665355442e88979b8b3c820ceed_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148163067/Put/seqid=0 2024-11-21T00:16:04,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742041_1217 (size=14994) 2024-11-21T00:16:04,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:04,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd834665355442e88979b8b3c820ceed_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd834665355442e88979b8b3c820ceed_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:04,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ba7a0dda8498d8e114a1aebeddf19, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:04,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ba7a0dda8498d8e114a1aebeddf19 is 175, key is test_row_0/A:col10/1732148163067/Put/seqid=0 2024-11-21T00:16:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742042_1218 (size=39949) 2024-11-21T00:16:04,304 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ba7a0dda8498d8e114a1aebeddf19 2024-11-21T00:16:04,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5737cd3d8c6d4d19aa960fa471acd194 is 50, key is test_row_0/B:col10/1732148163067/Put/seqid=0 2024-11-21T00:16:04,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742043_1219 (size=12301) 2024-11-21T00:16:04,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148224318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:04,367 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/f0c7d0ed09fc40bd8abf5c60ed305550 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f0c7d0ed09fc40bd8abf5c60ed305550 2024-11-21T00:16:04,368 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#175 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:04,368 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/397cb8221dc548469af9620739a4f8d3 is 175, key is test_row_0/A:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:04,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742044_1220 (size=32107) 2024-11-21T00:16:04,380 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into f0c7d0ed09fc40bd8abf5c60ed305550(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:04,380 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:04,380 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148163873; duration=0sec 2024-11-21T00:16:04,380 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:04,380 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:16:04,380 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:04,383 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:04,383 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:16:04,383 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:04,383 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/74b709a99923400a9a4ff69566431160, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.8 K 2024-11-21T00:16:04,384 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b709a99923400a9a4ff69566431160, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732148160102 2024-11-21T00:16:04,384 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2fdc5aef684a708df4b83e7b0cc348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148160190 2024-11-21T00:16:04,385 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 78484466cef94d3fbfb61a1872a62ae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:04,389 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/397cb8221dc548469af9620739a4f8d3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3 2024-11-21T00:16:04,394 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:04,394 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/7193cc2a7f3e4704ae111c1cc0dbcf1d is 50, key is test_row_0/C:col10/1732148162483/Put/seqid=0 2024-11-21T00:16:04,395 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 397cb8221dc548469af9620739a4f8d3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:04,395 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:04,395 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148163872; duration=0sec 2024-11-21T00:16:04,395 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:04,395 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:16:04,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742045_1221 (size=13153) 2024-11-21T00:16:04,407 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/7193cc2a7f3e4704ae111c1cc0dbcf1d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/7193cc2a7f3e4704ae111c1cc0dbcf1d 2024-11-21T00:16:04,412 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into 7193cc2a7f3e4704ae111c1cc0dbcf1d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:04,412 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:04,412 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148163873; duration=0sec 2024-11-21T00:16:04,412 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:04,412 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:16:04,419 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-21T00:16:04,419 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:04,419 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,419 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:04,421 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-21T00:16:04,421 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-21T00:16:04,421 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:04,421 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,421 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:04,421 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,421 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,421 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:04,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:04,432 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-21T00:16:04,432 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:04,432 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,432 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:04,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148224446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:04,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148224657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:16:04,666 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:04,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5737cd3d8c6d4d19aa960fa471acd194 2024-11-21T00:16:04,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/cb674680d23c4e86842ff4ffec12d758 is 50, key is test_row_0/C:col10/1732148163067/Put/seqid=0 2024-11-21T00:16:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742046_1222 (size=12301) 2024-11-21T00:16:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58990 deadline: 1732148224967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:05,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/cb674680d23c4e86842ff4ffec12d758 2024-11-21T00:16:05,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/3a6ba7a0dda8498d8e114a1aebeddf19 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19 2024-11-21T00:16:05,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19, entries=200, sequenceid=468, filesize=39.0 K 2024-11-21T00:16:05,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/5737cd3d8c6d4d19aa960fa471acd194 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194 2024-11-21T00:16:05,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194, entries=150, sequenceid=468, filesize=12.0 K 2024-11-21T00:16:05,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/cb674680d23c4e86842ff4ffec12d758 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758 2024-11-21T00:16:05,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758, entries=150, sequenceid=468, filesize=12.0 K 2024-11-21T00:16:05,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 913ms, sequenceid=468, compaction requested=false 2024-11-21T00:16:05,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:05,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T00:16:05,480 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-21T00:16:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:05,480 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:05,480 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:05,481 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:05,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:05,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213db8d906d50d475e8529dae13d7d97ca_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:05,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742047_1223 (size=12454) 2024-11-21T00:16:05,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:05,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213db8d906d50d475e8529dae13d7d97ca_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213db8d906d50d475e8529dae13d7d97ca_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:05,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1527200d2a884572a2792126ee7b4f0c, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:05,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1527200d2a884572a2792126ee7b4f0c is 175, key is test_row_0/A:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:05,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742048_1224 (size=31255) 2024-11-21T00:16:05,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1527200d2a884572a2792126ee7b4f0c 2024-11-21T00:16:05,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/85db8cb447c540fb8b36bedd5ba84bf4 is 50, key is test_row_0/B:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:05,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742049_1225 (size=12301) 2024-11-21T00:16:06,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/85db8cb447c540fb8b36bedd5ba84bf4 2024-11-21T00:16:06,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/e4c1fd37daa640568e5c16dc44e73bc5 is 50, key is test_row_0/C:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:06,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742050_1226 (size=12301) 2024-11-21T00:16:06,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/e4c1fd37daa640568e5c16dc44e73bc5 2024-11-21T00:16:06,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/1527200d2a884572a2792126ee7b4f0c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c 2024-11-21T00:16:06,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c, entries=150, sequenceid=483, filesize=30.5 K 2024-11-21T00:16:06,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/85db8cb447c540fb8b36bedd5ba84bf4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4 2024-11-21T00:16:06,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4, entries=150, sequenceid=483, filesize=12.0 K 2024-11-21T00:16:06,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/e4c1fd37daa640568e5c16dc44e73bc5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5 2024-11-21T00:16:06,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5, entries=150, sequenceid=483, filesize=12.0 K 2024-11-21T00:16:06,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1270ms, sequenceid=483, compaction requested=true 2024-11-21T00:16:06,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:06,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:06,750 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb31c4d3052e5d7cdaabfc9b52a02c70:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:06,750 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:06,750 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:06,750 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:06,750 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/B is initiating minor compaction (all files) 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/A is initiating minor compaction (all files) 2024-11-21T00:16:06,751 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/B in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:06,751 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/A in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:06,751 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f0c7d0ed09fc40bd8abf5c60ed305550, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.9 K 2024-11-21T00:16:06,751 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=100.9 K 2024-11-21T00:16:06,751 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c] 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c7d0ed09fc40bd8abf5c60ed305550, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397cb8221dc548469af9620739a4f8d3, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5737cd3d8c6d4d19aa960fa471acd194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732148163067 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6ba7a0dda8498d8e114a1aebeddf19, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732148163067 2024-11-21T00:16:06,751 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 85db8cb447c540fb8b36bedd5ba84bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732148164297 2024-11-21T00:16:06,752 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1527200d2a884572a2792126ee7b4f0c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732148164297 2024-11-21T00:16:06,759 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:06,759 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/637eb305ca004e568665dec516db0bb8 is 50, key is test_row_0/B:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:06,762 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:06,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742051_1227 (size=13255) 2024-11-21T00:16:06,764 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d22caa2783814fc4bb8d1d43395d6e95_cb31c4d3052e5d7cdaabfc9b52a02c70 store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:06,791 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d22caa2783814fc4bb8d1d43395d6e95_cb31c4d3052e5d7cdaabfc9b52a02c70, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:06,791 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d22caa2783814fc4bb8d1d43395d6e95_cb31c4d3052e5d7cdaabfc9b52a02c70 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:06,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742052_1228 (size=4469) 2024-11-21T00:16:07,173 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/637eb305ca004e568665dec516db0bb8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/637eb305ca004e568665dec516db0bb8 2024-11-21T00:16:07,179 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/B of cb31c4d3052e5d7cdaabfc9b52a02c70 into 637eb305ca004e568665dec516db0bb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:07,179 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:07,179 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/B, priority=13, startTime=1732148166750; duration=0sec 2024-11-21T00:16:07,179 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:07,179 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:B 2024-11-21T00:16:07,179 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:07,181 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:07,181 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): cb31c4d3052e5d7cdaabfc9b52a02c70/C is initiating minor compaction (all files) 2024-11-21T00:16:07,181 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb31c4d3052e5d7cdaabfc9b52a02c70/C in TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:07,181 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/7193cc2a7f3e4704ae111c1cc0dbcf1d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp, totalSize=36.9 K 2024-11-21T00:16:07,181 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 7193cc2a7f3e4704ae111c1cc0dbcf1d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732148162483 2024-11-21T00:16:07,182 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cb674680d23c4e86842ff4ffec12d758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732148163067 2024-11-21T00:16:07,182 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c1fd37daa640568e5c16dc44e73bc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732148164297 2024-11-21T00:16:07,192 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:07,192 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/2856f67771ce450caba7e804660b37b6 is 50, key is test_row_0/C:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:07,196 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb31c4d3052e5d7cdaabfc9b52a02c70#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742053_1229 (size=13255) 2024-11-21T00:16:07,197 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/7164c67e5cbe41cba2737782e541cedd is 175, key is test_row_0/A:col10/1732148164316/Put/seqid=0 2024-11-21T00:16:07,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742054_1230 (size=32209) 2024-11-21T00:16:07,643 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/7164c67e5cbe41cba2737782e541cedd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/7164c67e5cbe41cba2737782e541cedd 2024-11-21T00:16:07,643 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/2856f67771ce450caba7e804660b37b6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/2856f67771ce450caba7e804660b37b6 2024-11-21T00:16:07,648 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/C of cb31c4d3052e5d7cdaabfc9b52a02c70 into 2856f67771ce450caba7e804660b37b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:07,648 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb31c4d3052e5d7cdaabfc9b52a02c70/A of cb31c4d3052e5d7cdaabfc9b52a02c70 into 7164c67e5cbe41cba2737782e541cedd(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb31c4d3052e5d7cdaabfc9b52a02c70: 2024-11-21T00:16:07,648 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/A, priority=13, startTime=1732148166749; duration=0sec 2024-11-21T00:16:07,648 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., storeName=cb31c4d3052e5d7cdaabfc9b52a02c70/C, priority=13, startTime=1732148166750; duration=0sec 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:A 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:07,648 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb31c4d3052e5d7cdaabfc9b52a02c70:C 2024-11-21T00:16:08,498 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-21T00:16:08,498 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:08,498 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,498 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:08,517 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-21T00:16:08,517 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:08,517 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,518 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:08,548 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-21T00:16:08,548 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:08,548 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,548 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:08,559 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-21T00:16:08,559 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:08,559 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,559 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 218 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2275 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2163 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 978 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2934 rows 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 969 2024-11-21T00:16:08,559 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2907 rows 2024-11-21T00:16:08,559 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:16:08,559 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:16:08,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:08,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:08,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:16:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:08,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148168563"}]},"ts":"1732148168563"} 2024-11-21T00:16:08,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:16:08,565 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:16:08,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:16:08,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, UNASSIGN}] 2024-11-21T00:16:08,567 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, UNASSIGN 2024-11-21T00:16:08,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:08,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, UNASSIGN because future has completed 2024-11-21T00:16:08,570 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:16:08,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:08,722 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:08,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:16:08,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing cb31c4d3052e5d7cdaabfc9b52a02c70, disabling compactions & flushes 2024-11-21T00:16:08,722 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:08,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:08,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. after waiting 0 ms 2024-11-21T00:16:08,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:08,722 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing cb31c4d3052e5d7cdaabfc9b52a02c70 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=A 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=B 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb31c4d3052e5d7cdaabfc9b52a02c70, store=C 2024-11-21T00:16:08,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:08,730 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121740fd242951b4b878f4e61ab9ec75974_cb31c4d3052e5d7cdaabfc9b52a02c70 is 50, key is test_row_0/A:col10/1732148168516/Put/seqid=0 2024-11-21T00:16:08,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742055_1231 (size=12454) 2024-11-21T00:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:09,135 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:09,138 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121740fd242951b4b878f4e61ab9ec75974_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121740fd242951b4b878f4e61ab9ec75974_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:09,139 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/aeb9d89e551f469490f4db9c49cf6718, store: [table=TestAcidGuarantees family=A region=cb31c4d3052e5d7cdaabfc9b52a02c70] 2024-11-21T00:16:09,140 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/aeb9d89e551f469490f4db9c49cf6718 is 175, key is test_row_0/A:col10/1732148168516/Put/seqid=0 2024-11-21T00:16:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742056_1232 (size=31255) 2024-11-21T00:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:09,544 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=493, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/aeb9d89e551f469490f4db9c49cf6718 2024-11-21T00:16:09,557 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/ef73ed7bc34e4168b0b9f13da23dd292 is 50, key is test_row_0/B:col10/1732148168516/Put/seqid=0 2024-11-21T00:16:09,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742057_1233 (size=12301) 2024-11-21T00:16:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:09,963 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/ef73ed7bc34e4168b0b9f13da23dd292 2024-11-21T00:16:09,974 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/c22d8c111d444ce2ba2b626b2863b306 is 50, key is test_row_0/C:col10/1732148168516/Put/seqid=0 2024-11-21T00:16:09,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742058_1234 (size=12301) 2024-11-21T00:16:10,378 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/c22d8c111d444ce2ba2b626b2863b306 2024-11-21T00:16:10,385 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/A/aeb9d89e551f469490f4db9c49cf6718 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/aeb9d89e551f469490f4db9c49cf6718 2024-11-21T00:16:10,389 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/aeb9d89e551f469490f4db9c49cf6718, entries=150, sequenceid=493, filesize=30.5 K 2024-11-21T00:16:10,390 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/B/ef73ed7bc34e4168b0b9f13da23dd292 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ef73ed7bc34e4168b0b9f13da23dd292 2024-11-21T00:16:10,394 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ef73ed7bc34e4168b0b9f13da23dd292, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T00:16:10,395 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/.tmp/C/c22d8c111d444ce2ba2b626b2863b306 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c22d8c111d444ce2ba2b626b2863b306 2024-11-21T00:16:10,398 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c22d8c111d444ce2ba2b626b2863b306, entries=150, sequenceid=493, filesize=12.0 K 2024-11-21T00:16:10,399 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1677ms, sequenceid=493, compaction requested=false 2024-11-21T00:16:10,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c] to archive 2024-11-21T00:16:10,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:10,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/bac67a48381341ce8211b5b2d1b83bdd 2024-11-21T00:16:10,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a86a265e80d44e0fba6143e787011e8b 2024-11-21T00:16:10,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/a95172eb212149999dbf593a363ca4e7 2024-11-21T00:16:10,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/4845c8aab8ec47bbb0b3bcdcc7513fb2 2024-11-21T00:16:10,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/dc7fbbccf11848a2b2b07617cc38c5fd 2024-11-21T00:16:10,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6a651a7c7dc242628d956f458721547f 2024-11-21T00:16:10,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/30b00ef46e9c4904a566892a438c0bfd 2024-11-21T00:16:10,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/17fed02501154ae0b1243239e9bd773c 2024-11-21T00:16:10,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1789c63c9fa04b3b8a97ee5e7c896479 2024-11-21T00:16:10,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/33cb000f448a495eb45209ed1aae4022 2024-11-21T00:16:10,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3b311ebac59c4cd9919e49ff93e42170 2024-11-21T00:16:10,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/6d89992a40e648e1a02da03ec611b311 2024-11-21T00:16:10,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/c4e9ff702ba14867807acc11f464e001 2024-11-21T00:16:10,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ece6c9f504d848fdcfd3b8ee7b75f 2024-11-21T00:16:10,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/2c89407190144abdb783fdb5ff2669c1 2024-11-21T00:16:10,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/38e08f0245464e76813ede7a0e16e5c4 2024-11-21T00:16:10,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/fa5e2b80c64c448da9e028d41df03e75 2024-11-21T00:16:10,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b6d6f48f65a040c184eecfaf9ade02c3 2024-11-21T00:16:10,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/ad76c21f03a948ca82916b7a924993c5 2024-11-21T00:16:10,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/b1a16afe4cab4b85b64e34780a62a5ef 2024-11-21T00:16:10,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/352c75fb4028428eadbde001e6b28be8 2024-11-21T00:16:10,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/40fcb0c93ed2419f993269239e22ef12 2024-11-21T00:16:10,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/18bb9aa27a53481096b3dc6143d3442f 2024-11-21T00:16:10,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/751756e49ef54dd9a4904491e6c21f03 2024-11-21T00:16:10,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/946b5ce4a1fd47c08663e267eb7b8e12 2024-11-21T00:16:10,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/397cb8221dc548469af9620739a4f8d3 2024-11-21T00:16:10,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/3a6ba7a0dda8498d8e114a1aebeddf19 2024-11-21T00:16:10,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/1527200d2a884572a2792126ee7b4f0c 2024-11-21T00:16:10,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/6bc7f0bb1f5e43d1911c18460701eb2b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5e3086cb759641d6a0e1c2c246b6e997, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5b34800692e44e498661028ab32dc59c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/56f655c2a97349fdb274d5d8cf332a01, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ca75c65a7ca54360b56918ce74b53168, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/b0e32023111e4518b145da888095fabe, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/d32b87c01c424ff8abb0c8757aba7c65, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f0c7d0ed09fc40bd8abf5c60ed305550, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4] to archive 2024-11-21T00:16:10,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:10,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/73824e24c1864ad88450cbdb0920eed3 2024-11-21T00:16:10,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a07b28985384354a325c3d173ad8653 2024-11-21T00:16:10,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/6bc7f0bb1f5e43d1911c18460701eb2b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/6bc7f0bb1f5e43d1911c18460701eb2b 2024-11-21T00:16:10,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/2d8a51705c374cc99ae59dead8e4bc57 2024-11-21T00:16:10,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f790ee3366b148259f8a2e7288c47fb2 2024-11-21T00:16:10,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7745c76b25a04823b81c4fddd3c3fa27 2024-11-21T00:16:10,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5e3086cb759641d6a0e1c2c246b6e997 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5e3086cb759641d6a0e1c2c246b6e997 2024-11-21T00:16:10,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/eb7ac8634bc246e38afd72e055c83ad2 2024-11-21T00:16:10,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/fd209b500bae427293aceecd0cb24b4e 2024-11-21T00:16:10,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5b34800692e44e498661028ab32dc59c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5b34800692e44e498661028ab32dc59c 2024-11-21T00:16:10,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/badf580e5c9b4a21b81864c2f066a477 2024-11-21T00:16:10,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/70ba6d65f48e4748b6bd23ef9969ff4c 2024-11-21T00:16:10,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/56f655c2a97349fdb274d5d8cf332a01 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/56f655c2a97349fdb274d5d8cf332a01 2024-11-21T00:16:10,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3a69a776309942d3a469633ecde7ea35 2024-11-21T00:16:10,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/0089d10852064a13a694f5715aa21a61 2024-11-21T00:16:10,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ca75c65a7ca54360b56918ce74b53168 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ca75c65a7ca54360b56918ce74b53168 2024-11-21T00:16:10,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9e09e0b775fb47b88ce0a856f82caf5e 2024-11-21T00:16:10,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/03c8cb4a6f7547ed8e64764271ba29c2 2024-11-21T00:16:10,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/b0e32023111e4518b145da888095fabe to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/b0e32023111e4518b145da888095fabe 2024-11-21T00:16:10,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3cf9b8abf4704f4d8ef1f53b82076538 2024-11-21T00:16:10,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/3475c451315b4ad886016e9d0d863321 2024-11-21T00:16:10,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/d32b87c01c424ff8abb0c8757aba7c65 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/d32b87c01c424ff8abb0c8757aba7c65 2024-11-21T00:16:10,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/9c4071323e85486386a7df1d38af74b4 2024-11-21T00:16:10,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/cf2dfb99418942adbe0a04e523e99483 2024-11-21T00:16:10,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f0c7d0ed09fc40bd8abf5c60ed305550 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/f0c7d0ed09fc40bd8abf5c60ed305550 2024-11-21T00:16:10,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/7a1a56995b8944c796374eadb20ec7fc 2024-11-21T00:16:10,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/5737cd3d8c6d4d19aa960fa471acd194 2024-11-21T00:16:10,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/85db8cb447c540fb8b36bedd5ba84bf4 2024-11-21T00:16:10,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a0220edf0867478f9bfeaa90fb39cf06, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ac2757ec427c4f89b8ea980fe8529d2d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a5feb0644d8942c5894dfae2ee798ac9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/d6c82af463a448438fa6d3e651925db9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b6956131de924297a4ecb74b20fbce4b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c8fd1c03bacf475aba956b0339c1adfd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/74b709a99923400a9a4ff69566431160, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/7193cc2a7f3e4704ae111c1cc0dbcf1d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5] to archive 2024-11-21T00:16:10,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:10,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f294d6e9e7a3481899d48de10ec0ae51 2024-11-21T00:16:10,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/9169e5a5ece8441f9fe47aa8a7c09f3f 2024-11-21T00:16:10,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a0220edf0867478f9bfeaa90fb39cf06 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a0220edf0867478f9bfeaa90fb39cf06 2024-11-21T00:16:10,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b82c5788682c4b29a617510e1c446348 2024-11-21T00:16:10,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/065dfd09ccd3450985b606f14f6ce0d8 2024-11-21T00:16:10,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/dca611da13bd47cfa5a942c1f8e9785e 2024-11-21T00:16:10,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ac2757ec427c4f89b8ea980fe8529d2d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ac2757ec427c4f89b8ea980fe8529d2d 2024-11-21T00:16:10,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/232c221d26ab4eb1b75bf753233ba275 2024-11-21T00:16:10,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/0c22d557a8724b32885ae32856d5d80b 2024-11-21T00:16:10,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a5feb0644d8942c5894dfae2ee798ac9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a5feb0644d8942c5894dfae2ee798ac9 2024-11-21T00:16:10,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/8b37eb0778ea4014aa39e5c8e55c1092 2024-11-21T00:16:10,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/f13806efb67145ef83e6cf0a07da5e07 2024-11-21T00:16:10,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/d6c82af463a448438fa6d3e651925db9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/d6c82af463a448438fa6d3e651925db9 2024-11-21T00:16:10,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/ae1544a16b8f4ccd99348580d137294d 2024-11-21T00:16:10,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/97d64e8ba76947d4b7f86a630dac94c1 2024-11-21T00:16:10,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b6956131de924297a4ecb74b20fbce4b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/b6956131de924297a4ecb74b20fbce4b 2024-11-21T00:16:10,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/037218ceb9af4932b65ab899a340c295 2024-11-21T00:16:10,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/a692f1c681244f5c90dcf9ab679021b1 2024-11-21T00:16:10,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c8fd1c03bacf475aba956b0339c1adfd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c8fd1c03bacf475aba956b0339c1adfd 2024-11-21T00:16:10,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/5968680864474fdabf7b8893720e2803 2024-11-21T00:16:10,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/868b9b4a16394af39fdecbe63018e075 2024-11-21T00:16:10,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/74b709a99923400a9a4ff69566431160 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/74b709a99923400a9a4ff69566431160 2024-11-21T00:16:10,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/34eb4d70d162491180aff5dc81dc02d2 2024-11-21T00:16:10,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/6e2fdc5aef684a708df4b83e7b0cc348 2024-11-21T00:16:10,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/7193cc2a7f3e4704ae111c1cc0dbcf1d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/7193cc2a7f3e4704ae111c1cc0dbcf1d 2024-11-21T00:16:10,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/78484466cef94d3fbfb61a1872a62ae6 2024-11-21T00:16:10,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/cb674680d23c4e86842ff4ffec12d758 2024-11-21T00:16:10,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/e4c1fd37daa640568e5c16dc44e73bc5 2024-11-21T00:16:10,501 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits/496.seqid, newMaxSeqId=496, maxSeqId=4 2024-11-21T00:16:10,502 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. 2024-11-21T00:16:10,502 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for cb31c4d3052e5d7cdaabfc9b52a02c70: Waiting for close lock at 1732148168722Running coprocessor pre-close hooks at 1732148168722Disabling compacts and flushes for region at 1732148168722Disabling writes for close at 1732148168722Obtaining lock to block concurrent updates at 1732148168722Preparing flush snapshotting stores in cb31c4d3052e5d7cdaabfc9b52a02c70 at 1732148168722Finished memstore snapshotting TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732148168723 (+1 ms)Flushing stores of TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70. at 1732148168723Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/A: creating writer at 1732148168724 (+1 ms)Flushing mob file cb31c4d3052e5d7cdaabfc9b52a02c70/A: appending metadata at 1732148168730 (+6 ms)Flushing mob file cb31c4d3052e5d7cdaabfc9b52a02c70/A: closing flushed file at 1732148168730Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/A: appending metadata at 1732148169139 (+409 ms)Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/A: closing flushed file at 1732148169139Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/B: creating writer at 1732148169553 (+414 ms)Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/B: appending metadata at 1732148169557 (+4 ms)Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/B: closing flushed file at 1732148169557Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/C: creating writer at 1732148169971 (+414 ms)Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/C: appending metadata at 1732148169974 (+3 ms)Flushing cb31c4d3052e5d7cdaabfc9b52a02c70/C: closing flushed file at 1732148169974Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4912481c: reopening flushed file at 1732148170384 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@299fe9ef: reopening flushed file at 1732148170389 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34af78ca: reopening flushed file at 1732148170394 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cb31c4d3052e5d7cdaabfc9b52a02c70 in 1677ms, sequenceid=493, compaction requested=false at 1732148170399 (+5 ms)Writing region close event to WAL at 1732148170498 (+99 ms)Running coprocessor post-close hooks at 1732148170501 (+3 ms)Closed at 1732148170502 (+1 ms) 2024-11-21T00:16:10,503 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cb31c4d3052e5d7cdaabfc9b52a02c70, regionState=CLOSED 2024-11-21T00:16:10,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:10,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T00:16:10,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure cb31c4d3052e5d7cdaabfc9b52a02c70, server=5ed4808ef0e6,40705,1732148113864 in 1.9360 sec 2024-11-21T00:16:10,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T00:16:10,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb31c4d3052e5d7cdaabfc9b52a02c70, UNASSIGN in 1.9410 sec 2024-11-21T00:16:10,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T00:16:10,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9450 sec 2024-11-21T00:16:10,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148170513"}]},"ts":"1732148170513"} 2024-11-21T00:16:10,514 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:16:10,514 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:16:10,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-21T00:16:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T00:16:10,707 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:10,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:16:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,709 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T00:16:10,709 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,713 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,715 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits] 2024-11-21T00:16:10,718 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/7164c67e5cbe41cba2737782e541cedd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/7164c67e5cbe41cba2737782e541cedd 2024-11-21T00:16:10,719 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/aeb9d89e551f469490f4db9c49cf6718 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/A/aeb9d89e551f469490f4db9c49cf6718 2024-11-21T00:16:10,721 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/637eb305ca004e568665dec516db0bb8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/637eb305ca004e568665dec516db0bb8 2024-11-21T00:16:10,722 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ef73ed7bc34e4168b0b9f13da23dd292 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/B/ef73ed7bc34e4168b0b9f13da23dd292 2024-11-21T00:16:10,725 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/2856f67771ce450caba7e804660b37b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/2856f67771ce450caba7e804660b37b6 2024-11-21T00:16:10,726 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c22d8c111d444ce2ba2b626b2863b306 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/C/c22d8c111d444ce2ba2b626b2863b306 2024-11-21T00:16:10,729 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits/496.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70/recovered.edits/496.seqid 2024-11-21T00:16:10,730 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,730 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:16:10,730 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:16:10,732 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:16:10,741 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112100be0fa88f914f7c8acf8e6afb839566_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112100be0fa88f914f7c8acf8e6afb839566_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,745 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210e07c2adf25e420c847c441b4a093fb4_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210e07c2adf25e420c847c441b4a093fb4_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,746 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126065b033bf64faba1ae1c56c30e104d_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126065b033bf64faba1ae1c56c30e104d_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,748 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a0bb210484546f48b5db9e9243b8026_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a0bb210484546f48b5db9e9243b8026_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,749 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130ae2a34a0fc4e748164f0ac3ed7c13a_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130ae2a34a0fc4e748164f0ac3ed7c13a_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,751 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213c642ff542ab4107b609f52a2c99f141_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213c642ff542ab4107b609f52a2c99f141_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213db8d906d50d475e8529dae13d7d97ca_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213db8d906d50d475e8529dae13d7d97ca_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213fafeaaee41f4e6bba050c3f4e720aeb_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213fafeaaee41f4e6bba050c3f4e720aeb_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,758 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150a89711904643f8bbdf2f502c7f2413_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150a89711904643f8bbdf2f502c7f2413_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,760 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c6ea8d155e54e65bd997eb103ce4847_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215c6ea8d155e54e65bd997eb103ce4847_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112160ad5f14aac14f7ca79884985c20e26e_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112160ad5f14aac14f7ca79884985c20e26e_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,763 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121740fd242951b4b878f4e61ab9ec75974_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121740fd242951b4b878f4e61ab9ec75974_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,764 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188e01bdffd7f48e081963904176b9a02_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188e01bdffd7f48e081963904176b9a02_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,765 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121966067e2127446aaa101aee032c79b95_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121966067e2127446aaa101aee032c79b95_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,767 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc83e88b7e246da862981e06f188891_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc83e88b7e246da862981e06f188891_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,768 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af041cb24a12414392a0782445ee58a3_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af041cb24a12414392a0782445ee58a3_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,769 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1ee51c2a8e24026b102e5549f8bdd8e_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1ee51c2a8e24026b102e5549f8bdd8e_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,770 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd834665355442e88979b8b3c820ceed_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd834665355442e88979b8b3c820ceed_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,778 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6bf9de9a6e54d7c8582c69d4a24af46_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6bf9de9a6e54d7c8582c69d4a24af46_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,779 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7e7267660dd4ce193b1943c6cb6a88a_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7e7267660dd4ce193b1943c6cb6a88a_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,781 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2cb71f4cb52430bb6f4836ac3d1c658_cb31c4d3052e5d7cdaabfc9b52a02c70 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2cb71f4cb52430bb6f4836ac3d1c658_cb31c4d3052e5d7cdaabfc9b52a02c70 2024-11-21T00:16:10,782 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:16:10,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,786 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:16:10,788 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:16:10,790 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,790 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:16:10,790 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148170790"}]},"ts":"9223372036854775807"} 2024-11-21T00:16:10,793 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:16:10,793 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cb31c4d3052e5d7cdaabfc9b52a02c70, NAME => 'TestAcidGuarantees,,1732148143068.cb31c4d3052e5d7cdaabfc9b52a02c70.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:16:10,793 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:16:10,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148170793"}]},"ts":"9223372036854775807"} 2024-11-21T00:16:10,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:16:10,796 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-21T00:16:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T00:16:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:16:10,817 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:10,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=254 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_823068347_22 at /127.0.0.1:39916 [Waiting for operation #735] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_823068347_22 at /127.0.0.1:42162 [Waiting for operation #664] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1977616121_22 at /127.0.0.1:40044 [Waiting for operation #716] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1977616121_22 at /127.0.0.1:54148 [Waiting for operation #411] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=877 (was 821) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=762 (was 1842) 2024-11-21T00:16:10,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=254, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=877, ProcessCount=11, AvailableMemoryMB=761 2024-11-21T00:16:10,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:16:10,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:16:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:10,844 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:16:10,844 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:10,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-21T00:16:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:10,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:16:10,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742059_1235 (size=1061) 2024-11-21T00:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:11,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:16:11,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742060_1236 (size=53) 2024-11-21T00:16:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ec77d7e5be4bbf0794947ca9843b822c, disabling compactions & flushes 2024-11-21T00:16:11,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. after waiting 0 ms 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:11,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:11,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ec77d7e5be4bbf0794947ca9843b822c: Waiting for close lock at 1732148171664Disabling compacts and flushes for region at 1732148171664Disabling writes for close at 1732148171664Writing region close event to WAL at 1732148171664Closed at 1732148171664 2024-11-21T00:16:11,671 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:16:11,671 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148171671"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148171671"}]},"ts":"1732148171671"} 2024-11-21T00:16:11,673 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:16:11,676 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:16:11,676 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148171676"}]},"ts":"1732148171676"} 2024-11-21T00:16:11,680 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:16:11,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, ASSIGN}] 2024-11-21T00:16:11,683 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, ASSIGN 2024-11-21T00:16:11,685 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:16:11,836 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=ec77d7e5be4bbf0794947ca9843b822c, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:11,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, ASSIGN because future has completed 2024-11-21T00:16:11,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:11,994 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:11,994 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:16:11,995 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:11,995 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:16:11,995 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:11,995 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:11,996 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:11,999 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:11,999 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec77d7e5be4bbf0794947ca9843b822c columnFamilyName A 2024-11-21T00:16:11,999 DEBUG [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:12,000 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(327): Store=ec77d7e5be4bbf0794947ca9843b822c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:12,000 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,001 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:12,001 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec77d7e5be4bbf0794947ca9843b822c columnFamilyName B 2024-11-21T00:16:12,002 DEBUG [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:12,002 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(327): Store=ec77d7e5be4bbf0794947ca9843b822c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:12,002 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,004 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:12,004 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec77d7e5be4bbf0794947ca9843b822c columnFamilyName C 2024-11-21T00:16:12,004 DEBUG [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:12,004 INFO [StoreOpener-ec77d7e5be4bbf0794947ca9843b822c-1 {}] regionserver.HStore(327): Store=ec77d7e5be4bbf0794947ca9843b822c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:12,005 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:12,005 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,005 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,006 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,007 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,007 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,007 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:16:12,008 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,011 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:16:12,011 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened ec77d7e5be4bbf0794947ca9843b822c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62244311, jitterRate=-0.0724874883890152}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:16:12,011 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:12,012 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for ec77d7e5be4bbf0794947ca9843b822c: Running coprocessor pre-open hook at 1732148171995Writing region info on filesystem at 1732148171995Initializing all the Stores at 1732148171996 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148171996Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148171996Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148171996Cleaning up temporary data from old regions at 1732148172007 (+11 ms)Running coprocessor post-open hooks at 1732148172011 (+4 ms)Region opened successfully at 1732148172012 (+1 ms) 2024-11-21T00:16:12,013 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., pid=63, masterSystemTime=1732148171991 2024-11-21T00:16:12,014 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:12,014 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:12,014 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=ec77d7e5be4bbf0794947ca9843b822c, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:12,015 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39691 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=5ed4808ef0e6,40705,1732148113864, table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T00:16:12,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:12,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T00:16:12,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 in 177 msec 2024-11-21T00:16:12,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T00:16:12,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, ASSIGN in 338 msec 2024-11-21T00:16:12,020 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:16:12,020 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148172020"}]},"ts":"1732148172020"} 2024-11-21T00:16:12,021 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:16:12,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:16:12,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-21T00:16:12,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:16:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T00:16:13,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:13,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12616921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8c4787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,011 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4329aad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,012 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:16:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:16:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@645ec3db 2024-11-21T00:16:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:16:13,017 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:16:13,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656c77b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ac24339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d388952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9efb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,027 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300d7349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbebaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,032 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d622a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cc7ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,038 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510d9e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba6b12d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d156a7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,046 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ccb213b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c81c0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,050 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a999c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,058 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab69000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,060 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d32847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,062 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5609223c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,064 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464b4df2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:13,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:13,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cd5469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:13,068 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4063bff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:13,069 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,069 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,069 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,070 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,069 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,070 DEBUG [Thread-1115 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,071 DEBUG [Thread-1115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,071 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,074 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,076 DEBUG [Thread-1113 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,076 DEBUG [Thread-1113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,080 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,092 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T00:16:13,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,100 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:13,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:13,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:13,109 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:13,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:16:13,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/762f3c538ff14c6a87957f3a35fcb6ca is 50, key is test_row_0/A:col10/1732148173101/Put/seqid=0 2024-11-21T00:16:13,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732148173133, max timestamp=1732148173158Num uniques 859; 2024-11-21T00:16:13,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:13,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148173133, max timestamp=1732148173158Num uniques 859; 2024-11-21T00:16:13,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:13,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148173155, max timestamp=1732148173171Num uniques 859; 2024-11-21T00:16:13,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:13,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742061_1237 (size=12001) 2024-11-21T00:16:13,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/762f3c538ff14c6a87957f3a35fcb6ca 2024-11-21T00:16:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:13,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/0c5e37aba9b24b4d8082ee4460baf86b is 50, key is test_row_0/B:col10/1732148173101/Put/seqid=0 2024-11-21T00:16:13,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742062_1238 (size=12001) 2024-11-21T00:16:13,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/0c5e37aba9b24b4d8082ee4460baf86b 2024-11-21T00:16:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148233212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148233219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148233256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148233256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148233256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6708abb5c53d44afaa042bbedc4e57b6 is 50, key is test_row_0/C:col10/1732148173101/Put/seqid=0 2024-11-21T00:16:13,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742063_1239 (size=12001) 2024-11-21T00:16:13,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6708abb5c53d44afaa042bbedc4e57b6 2024-11-21T00:16:13,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/762f3c538ff14c6a87957f3a35fcb6ca as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca 2024-11-21T00:16:13,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:16:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/0c5e37aba9b24b4d8082ee4460baf86b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b 2024-11-21T00:16:13,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:16:13,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6708abb5c53d44afaa042bbedc4e57b6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6 2024-11-21T00:16:13,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:16:13,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for ec77d7e5be4bbf0794947ca9843b822c in 241ms, sequenceid=13, compaction requested=false 2024-11-21T00:16:13,342 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T00:16:13,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:13,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:13,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e3e8b3d192a44eab96ef36ae7cb37edd is 50, key is test_row_0/A:col10/1732148173372/Put/seqid=0 2024-11-21T00:16:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148233387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148233387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148233394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148233394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148233395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:13,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742064_1240 (size=16681) 2024-11-21T00:16:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148233506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148233506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148233507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148233507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148233525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148233719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148233719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148233729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148233729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148233736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:13,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e3e8b3d192a44eab96ef36ae7cb37edd 2024-11-21T00:16:13,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/fdc474587f124a4eb59918dc1c9c3507 is 50, key is test_row_0/B:col10/1732148173372/Put/seqid=0 2024-11-21T00:16:13,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:13,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:13,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742065_1241 (size=12001) 2024-11-21T00:16:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148234037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148234037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148234038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148234047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148234047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:14,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:14,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/fdc474587f124a4eb59918dc1c9c3507 2024-11-21T00:16:14,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6662bb838f6043f1818628d8174f6ceb is 50, key is test_row_0/C:col10/1732148173372/Put/seqid=0 2024-11-21T00:16:14,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742066_1242 (size=12001) 2024-11-21T00:16:14,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148234547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148234547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148234547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148234557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148234566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:14,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:14,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6662bb838f6043f1818628d8174f6ceb 2024-11-21T00:16:14,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e3e8b3d192a44eab96ef36ae7cb37edd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd 2024-11-21T00:16:14,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd, entries=250, sequenceid=50, filesize=16.3 K 2024-11-21T00:16:14,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/fdc474587f124a4eb59918dc1c9c3507 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507 2024-11-21T00:16:14,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T00:16:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6662bb838f6043f1818628d8174f6ceb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb 2024-11-21T00:16:14,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T00:16:14,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for ec77d7e5be4bbf0794947ca9843b822c in 1455ms, sequenceid=50, compaction requested=false 2024-11-21T00:16:14,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:14,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:16:14,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:14,845 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9ac8405d98f04c6eb94b02d24742cdb1 is 50, key is test_row_1/A:col10/1732148173393/Put/seqid=0 2024-11-21T00:16:14,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742067_1243 (size=9657) 2024-11-21T00:16:14,908 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9ac8405d98f04c6eb94b02d24742cdb1 2024-11-21T00:16:14,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/d6c2088174fb496496efe14ad04207f1 is 50, key is test_row_1/B:col10/1732148173393/Put/seqid=0 2024-11-21T00:16:14,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742068_1244 (size=9657) 2024-11-21T00:16:14,960 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/d6c2088174fb496496efe14ad04207f1 2024-11-21T00:16:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a93f6df64d8144319d421ef50b7f9737 is 50, key is test_row_1/C:col10/1732148173393/Put/seqid=0 2024-11-21T00:16:15,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742069_1245 (size=9657) 2024-11-21T00:16:15,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a93f6df64d8144319d421ef50b7f9737 2024-11-21T00:16:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9ac8405d98f04c6eb94b02d24742cdb1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1 2024-11-21T00:16:15,056 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1, entries=100, sequenceid=59, filesize=9.4 K 2024-11-21T00:16:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/d6c2088174fb496496efe14ad04207f1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1 2024-11-21T00:16:15,066 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1, entries=100, sequenceid=59, filesize=9.4 K 2024-11-21T00:16:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a93f6df64d8144319d421ef50b7f9737 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737 2024-11-21T00:16:15,073 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737, entries=100, sequenceid=59, filesize=9.4 K 2024-11-21T00:16:15,077 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for ec77d7e5be4bbf0794947ca9843b822c in 232ms, sequenceid=59, compaction requested=true 2024-11-21T00:16:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T00:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T00:16:15,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T00:16:15,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9760 sec 2024-11-21T00:16:15,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9890 sec 2024-11-21T00:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:16:15,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:15,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T00:16:15,250 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:16:15,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:15,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:16:15,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:16:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T00:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T00:16:15,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T00:16:15,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 160 msec 2024-11-21T00:16:15,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 169 msec 2024-11-21T00:16:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:16:15,566 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:15,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T00:16:15,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:15,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:15,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:15,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:15,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:15,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/28badd63158e4b829ae2ffd527e8c9f4 is 50, key is test_row_0/A:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:15,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742070_1246 (size=12001) 2024-11-21T00:16:15,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/28badd63158e4b829ae2ffd527e8c9f4 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148175611, max timestamp=1732148175635Num uniques 859; 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148175611, max timestamp=1732148175635Num uniques 859; 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148175611, max timestamp=1732148175635Num uniques 859; 2024-11-21T00:16:15,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:15,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/ac70468516474864ac1999524db876ab is 50, key is test_row_0/B:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148235669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148235669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148235676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742071_1247 (size=12001) 2024-11-21T00:16:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148235678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148235677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148235787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148235787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148235797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148235797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148235797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:15,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:15,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:16,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148235996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148236004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148236007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148236007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148236007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,055 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:16:16,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/ac70468516474864ac1999524db876ab 2024-11-21T00:16:16,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/bf9e846a52e6455ab01b8599a034f4e5 is 50, key is test_row_0/C:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:16,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:16,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742072_1248 (size=12001) 2024-11-21T00:16:16,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148236317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148236317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148236326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148236326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148236327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/bf9e846a52e6455ab01b8599a034f4e5 2024-11-21T00:16:16,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/28badd63158e4b829ae2ffd527e8c9f4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4 2024-11-21T00:16:16,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:16,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T00:16:16,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/ac70468516474864ac1999524db876ab as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab 2024-11-21T00:16:16,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T00:16:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/bf9e846a52e6455ab01b8599a034f4e5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5 2024-11-21T00:16:16,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T00:16:16,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ec77d7e5be4bbf0794947ca9843b822c in 1113ms, sequenceid=71, compaction requested=true 2024-11-21T00:16:16,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:16,691 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:16,691 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:16,692 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:16,692 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:16,692 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:16,692 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:16,693 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,693 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,693 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=44.6 K 2024-11-21T00:16:16,693 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=49.2 K 2024-11-21T00:16:16,694 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5e37aba9b24b4d8082ee4460baf86b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148173087 2024-11-21T00:16:16,694 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 762f3c538ff14c6a87957f3a35fcb6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148173087 2024-11-21T00:16:16,694 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc474587f124a4eb59918dc1c9c3507, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148173210 2024-11-21T00:16:16,694 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e8b3d192a44eab96ef36ae7cb37edd, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148173210 2024-11-21T00:16:16,695 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6c2088174fb496496efe14ad04207f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148173385 2024-11-21T00:16:16,695 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac8405d98f04c6eb94b02d24742cdb1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148173385 2024-11-21T00:16:16,695 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac70468516474864ac1999524db876ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:16,697 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 28badd63158e4b829ae2ffd527e8c9f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:16,715 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#201 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:16,715 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/2b9077aaa9b84ef1926b692849e1b282 is 50, key is test_row_0/B:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:16,728 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:16,728 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/7f5b6864af8b458c9b45efe866d68bf2 is 50, key is test_row_0/A:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742074_1250 (size=12139) 2024-11-21T00:16:16,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742073_1249 (size=12139) 2024-11-21T00:16:16,773 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/7f5b6864af8b458c9b45efe866d68bf2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/7f5b6864af8b458c9b45efe866d68bf2 2024-11-21T00:16:16,778 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 7f5b6864af8b458c9b45efe866d68bf2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:16,778 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:16,778 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=12, startTime=1732148176690; duration=0sec 2024-11-21T00:16:16,778 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:16,778 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:16,779 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:16,782 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:16,782 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:16,782 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,782 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=44.6 K 2024-11-21T00:16:16,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6708abb5c53d44afaa042bbedc4e57b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148173087 2024-11-21T00:16:16,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6662bb838f6043f1818628d8174f6ceb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148173210 2024-11-21T00:16:16,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a93f6df64d8144319d421ef50b7f9737, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148173385 2024-11-21T00:16:16,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9e846a52e6455ab01b8599a034f4e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:16,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:16:16,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:16,816 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T00:16:16,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:16,825 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#203 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:16:16,828 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/019e0223f95f4a27b9c8b16c0c50fdbd is 50, key is test_row_0/C:col10/1732148175576/Put/seqid=0 2024-11-21T00:16:16,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/cc4d7c1c346347e2a31d8ef531744a88 is 50, key is test_row_0/A:col10/1732148175662/Put/seqid=0 2024-11-21T00:16:16,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:16,872 DEBUG [regionserver/5ed4808ef0e6:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-21T00:16:16,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148236867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148236868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148236873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742075_1251 (size=12139) 2024-11-21T00:16:16,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148236876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148236879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,893 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/019e0223f95f4a27b9c8b16c0c50fdbd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/019e0223f95f4a27b9c8b16c0c50fdbd 2024-11-21T00:16:16,900 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 019e0223f95f4a27b9c8b16c0c50fdbd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:16,900 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:16,900 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=12, startTime=1732148176691; duration=0sec 2024-11-21T00:16:16,900 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:16,900 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:16,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742076_1252 (size=12001) 2024-11-21T00:16:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148236987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148236987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148236987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148236987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148236997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148237198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148237198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148237198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,209 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/2b9077aaa9b84ef1926b692849e1b282 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2b9077aaa9b84ef1926b692849e1b282 2024-11-21T00:16:17,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148237207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148237207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,234 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into 2b9077aaa9b84ef1926b692849e1b282(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:17,234 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:17,234 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=12, startTime=1732148176691; duration=0sec 2024-11-21T00:16:17,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:17,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:17,309 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/cc4d7c1c346347e2a31d8ef531744a88 2024-11-21T00:16:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/add79d74e99d478393bbcc9a4424a942 is 50, key is test_row_0/B:col10/1732148175662/Put/seqid=0 2024-11-21T00:16:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742077_1253 (size=12001) 2024-11-21T00:16:17,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148237506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148237507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148237507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148237516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148237516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:17,795 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/add79d74e99d478393bbcc9a4424a942 2024-11-21T00:16:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/ce662d7153a54aa98ccccc99f1f1625c is 50, key is test_row_0/C:col10/1732148175662/Put/seqid=0 2024-11-21T00:16:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742078_1254 (size=12001) 2024-11-21T00:16:17,839 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/ce662d7153a54aa98ccccc99f1f1625c 2024-11-21T00:16:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/cc4d7c1c346347e2a31d8ef531744a88 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88 2024-11-21T00:16:17,863 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88, entries=150, sequenceid=107, filesize=11.7 K 2024-11-21T00:16:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/add79d74e99d478393bbcc9a4424a942 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942 2024-11-21T00:16:17,888 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942, entries=150, sequenceid=107, filesize=11.7 K 2024-11-21T00:16:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/ce662d7153a54aa98ccccc99f1f1625c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c 2024-11-21T00:16:17,900 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c, entries=150, sequenceid=107, filesize=11.7 K 2024-11-21T00:16:17,901 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ec77d7e5be4bbf0794947ca9843b822c in 1085ms, sequenceid=107, compaction requested=false 2024-11-21T00:16:17,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:17,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:17,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T00:16:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T00:16:17,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T00:16:17,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-21T00:16:17,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3390 sec 2024-11-21T00:16:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:18,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5ed89db690f3431b84f86ccbe7234e0d is 50, key is test_row_0/A:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148238055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742079_1255 (size=14341) 2024-11-21T00:16:18,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5ed89db690f3431b84f86ccbe7234e0d 2024-11-21T00:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148238061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148238061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148238063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148238063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/3938561a199b41ebb3c5e2373cd71cb0 is 50, key is test_row_0/B:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742080_1256 (size=12001) 2024-11-21T00:16:18,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/3938561a199b41ebb3c5e2373cd71cb0 2024-11-21T00:16:18,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/be42db6aab3748afac5f4774ec0a0c6b is 50, key is test_row_0/C:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148238167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742081_1257 (size=12001) 2024-11-21T00:16:18,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/be42db6aab3748afac5f4774ec0a0c6b 2024-11-21T00:16:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148238177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148238177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148238178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148238178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5ed89db690f3431b84f86ccbe7234e0d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d 2024-11-21T00:16:18,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d, entries=200, sequenceid=123, filesize=14.0 K 2024-11-21T00:16:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/3938561a199b41ebb3c5e2373cd71cb0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0 2024-11-21T00:16:18,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0, entries=150, sequenceid=123, filesize=11.7 K 2024-11-21T00:16:18,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/be42db6aab3748afac5f4774ec0a0c6b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b 2024-11-21T00:16:18,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b, entries=150, sequenceid=123, filesize=11.7 K 2024-11-21T00:16:18,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec77d7e5be4bbf0794947ca9843b822c in 202ms, sequenceid=123, compaction requested=true 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:18,231 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:18,231 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:18,233 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:18,233 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:18,233 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:18,233 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/7f5b6864af8b458c9b45efe866d68bf2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=37.6 K 2024-11-21T00:16:18,234 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5b6864af8b458c9b45efe866d68bf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:18,234 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4d7c1c346347e2a31d8ef531744a88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732148175662 2024-11-21T00:16:18,235 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:18,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed89db690f3431b84f86ccbe7234e0d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:18,235 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:18,235 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:18,235 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2b9077aaa9b84ef1926b692849e1b282, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=35.3 K 2024-11-21T00:16:18,236 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9077aaa9b84ef1926b692849e1b282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:18,236 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting add79d74e99d478393bbcc9a4424a942, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732148175662 2024-11-21T00:16:18,237 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3938561a199b41ebb3c5e2373cd71cb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:18,263 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:18,264 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/78c2fee9b78142a2b3af4b0665fd4d95 is 50, key is test_row_0/B:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,268 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:18,269 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9185018f03ec4304880a1c36d3b7e350 is 50, key is test_row_0/A:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742082_1258 (size=12241) 2024-11-21T00:16:18,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742083_1259 (size=12241) 2024-11-21T00:16:18,363 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9185018f03ec4304880a1c36d3b7e350 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9185018f03ec4304880a1c36d3b7e350 2024-11-21T00:16:18,370 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 9185018f03ec4304880a1c36d3b7e350(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:18,370 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:18,371 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148178231; duration=0sec 2024-11-21T00:16:18,371 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:18,371 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:18,371 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:18,372 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:18,372 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:18,372 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:18,372 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/019e0223f95f4a27b9c8b16c0c50fdbd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=35.3 K 2024-11-21T00:16:18,372 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019e0223f95f4a27b9c8b16c0c50fdbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732148175568 2024-11-21T00:16:18,374 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce662d7153a54aa98ccccc99f1f1625c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732148175662 2024-11-21T00:16:18,375 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting be42db6aab3748afac5f4774ec0a0c6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:18,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:18,407 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:18,408 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2e21c4d53f2947ad9bd11bd0f6774a0b is 50, key is test_row_0/C:col10/1732148176873/Put/seqid=0 2024-11-21T00:16:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148238406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148238408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148238408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148238410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/c2a6281f306040a39c6fb7556abd7747 is 50, key is test_row_0/A:col10/1732148178392/Put/seqid=0 2024-11-21T00:16:18,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148238414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742084_1260 (size=12241) 2024-11-21T00:16:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742085_1261 (size=16931) 2024-11-21T00:16:18,454 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2e21c4d53f2947ad9bd11bd0f6774a0b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2e21c4d53f2947ad9bd11bd0f6774a0b 2024-11-21T00:16:18,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/c2a6281f306040a39c6fb7556abd7747 2024-11-21T00:16:18,470 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 2e21c4d53f2947ad9bd11bd0f6774a0b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:18,470 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:18,470 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148178231; duration=0sec 2024-11-21T00:16:18,471 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:18,471 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:18,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/179077e9f2ac4187b01a22b637db113e is 50, key is test_row_0/B:col10/1732148178392/Put/seqid=0 2024-11-21T00:16:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742086_1262 (size=12151) 2024-11-21T00:16:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148238529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148238529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148238530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148238530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148238532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148238739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148238740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148238752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148238752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148238752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,763 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/78c2fee9b78142a2b3af4b0665fd4d95 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/78c2fee9b78142a2b3af4b0665fd4d95 2024-11-21T00:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:18,786 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into 78c2fee9b78142a2b3af4b0665fd4d95(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:18,786 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:18,786 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148178231; duration=0sec 2024-11-21T00:16:18,786 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:18,786 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:18,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/179077e9f2ac4187b01a22b637db113e 2024-11-21T00:16:18,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b1b6cd7df174d7e941f0c2e839cfc36 is 50, key is test_row_0/C:col10/1732148178392/Put/seqid=0 2024-11-21T00:16:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742087_1263 (size=12151) 2024-11-21T00:16:18,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b1b6cd7df174d7e941f0c2e839cfc36 2024-11-21T00:16:18,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/c2a6281f306040a39c6fb7556abd7747 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747 2024-11-21T00:16:18,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747, entries=250, sequenceid=144, filesize=16.5 K 2024-11-21T00:16:19,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/179077e9f2ac4187b01a22b637db113e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e 2024-11-21T00:16:19,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e, entries=150, sequenceid=144, filesize=11.9 K 2024-11-21T00:16:19,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b1b6cd7df174d7e941f0c2e839cfc36 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36 2024-11-21T00:16:19,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36, entries=150, sequenceid=144, filesize=11.9 K 2024-11-21T00:16:19,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ec77d7e5be4bbf0794947ca9843b822c in 637ms, sequenceid=144, compaction requested=false 2024-11-21T00:16:19,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:19,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:19,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5f92f5bec52645c890d18998e51903de is 50, key is test_row_0/A:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148239123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148239124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148239126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148239129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148239126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742088_1264 (size=12151) 2024-11-21T00:16:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148239251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148239251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148239252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148239252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148239252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148239467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148239467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148239467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148239467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148239467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5f92f5bec52645c890d18998e51903de 2024-11-21T00:16:19,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/6cc9d7d52ad64cf4b1b7728c18950bcb is 50, key is test_row_0/B:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742089_1265 (size=12151) 2024-11-21T00:16:19,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/6cc9d7d52ad64cf4b1b7728c18950bcb 2024-11-21T00:16:19,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/0ca6162230f64bb289473525331a1de3 is 50, key is test_row_0/C:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742090_1266 (size=12151) 2024-11-21T00:16:19,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/0ca6162230f64bb289473525331a1de3 2024-11-21T00:16:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5f92f5bec52645c890d18998e51903de as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de 2024-11-21T00:16:19,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T00:16:19,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/6cc9d7d52ad64cf4b1b7728c18950bcb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb 2024-11-21T00:16:19,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T00:16:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/0ca6162230f64bb289473525331a1de3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3 2024-11-21T00:16:19,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T00:16:19,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ec77d7e5be4bbf0794947ca9843b822c in 636ms, sequenceid=160, compaction requested=true 2024-11-21T00:16:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:19,704 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:19,704 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:19,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:19,708 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:19,709 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:19,709 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:19,709 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/78c2fee9b78142a2b3af4b0665fd4d95, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=35.7 K 2024-11-21T00:16:19,709 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c2fee9b78142a2b3af4b0665fd4d95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:19,710 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 179077e9f2ac4187b01a22b637db113e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732148178061 2024-11-21T00:16:19,711 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc9d7d52ad64cf4b1b7728c18950bcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:19,712 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:19,712 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:19,712 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9185018f03ec4304880a1c36d3b7e350, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=40.4 K 2024-11-21T00:16:19,715 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9185018f03ec4304880a1c36d3b7e350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:19,715 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a6281f306040a39c6fb7556abd7747, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732148178058 2024-11-21T00:16:19,716 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f92f5bec52645c890d18998e51903de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:16:19,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:19,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:19,738 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:19,739 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9c8338c7b33f41f7aeba201857ae23da is 50, key is test_row_0/B:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T00:16:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:19,745 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:19,746 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#220 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:19,746 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:19,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:19,746 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/8ab927a75f8c41c2a3cbdd353fdf34cd is 50, key is test_row_0/A:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742091_1267 (size=12493) 2024-11-21T00:16:19,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742092_1268 (size=12493) 2024-11-21T00:16:19,773 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/8ab927a75f8c41c2a3cbdd353fdf34cd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8ab927a75f8c41c2a3cbdd353fdf34cd 2024-11-21T00:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:19,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T00:16:19,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:19,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:19,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:19,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:19,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:19,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:19,785 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 8ab927a75f8c41c2a3cbdd353fdf34cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:19,785 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:19,785 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148179704; duration=0sec 2024-11-21T00:16:19,785 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:19,786 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:19,786 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:19,787 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:19,787 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:19,787 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:19,787 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2e21c4d53f2947ad9bd11bd0f6774a0b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=35.7 K 2024-11-21T00:16:19,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/113cee5e598b48d3abc920a35fbf0ec2 is 50, key is test_row_0/A:col10/1732148179119/Put/seqid=0 2024-11-21T00:16:19,787 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e21c4d53f2947ad9bd11bd0f6774a0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732148176873 2024-11-21T00:16:19,787 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1b6cd7df174d7e941f0c2e839cfc36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732148178061 2024-11-21T00:16:19,788 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca6162230f64bb289473525331a1de3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:19,797 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:19,797 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/645303fbe9a14589ba758db1e404ea07 is 50, key is test_row_0/C:col10/1732148179062/Put/seqid=0 2024-11-21T00:16:19,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148239789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148239790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148239791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148239796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148239797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742093_1269 (size=16931) 2024-11-21T00:16:19,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/113cee5e598b48d3abc920a35fbf0ec2 2024-11-21T00:16:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742094_1270 (size=12493) 2024-11-21T00:16:19,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1c81efcb1cc349619ff7d4ce82bce657 is 50, key is test_row_0/B:col10/1732148179119/Put/seqid=0 2024-11-21T00:16:19,836 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/645303fbe9a14589ba758db1e404ea07 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/645303fbe9a14589ba758db1e404ea07 2024-11-21T00:16:19,842 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 645303fbe9a14589ba758db1e404ea07(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:19,842 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:19,842 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148179704; duration=0sec 2024-11-21T00:16:19,842 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:19,842 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:19,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742095_1271 (size=12151) 2024-11-21T00:16:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:19,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1c81efcb1cc349619ff7d4ce82bce657 2024-11-21T00:16:19,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/867a64bf4935455fbcab598f7b16d07f is 50, key is test_row_0/C:col10/1732148179119/Put/seqid=0 2024-11-21T00:16:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742096_1272 (size=12151) 2024-11-21T00:16:19,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:16:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:19,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148239906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148239906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148239907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148239907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:19,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148239916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:16:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:20,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148240116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148240117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148240117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148240117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148240126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,169 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9c8338c7b33f41f7aeba201857ae23da as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9c8338c7b33f41f7aeba201857ae23da 2024-11-21T00:16:20,175 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into 9c8338c7b33f41f7aeba201857ae23da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:20,177 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:20,177 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148179704; duration=0sec 2024-11-21T00:16:20,177 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:20,177 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:20,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:16:20,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:20,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:20,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:20,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:20,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/867a64bf4935455fbcab598f7b16d07f 2024-11-21T00:16:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/113cee5e598b48d3abc920a35fbf0ec2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2 2024-11-21T00:16:20,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2, entries=250, sequenceid=181, filesize=16.5 K 2024-11-21T00:16:20,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1c81efcb1cc349619ff7d4ce82bce657 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657 2024-11-21T00:16:20,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657, entries=150, sequenceid=181, filesize=11.9 K 2024-11-21T00:16:20,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/867a64bf4935455fbcab598f7b16d07f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f 2024-11-21T00:16:20,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f, entries=150, sequenceid=181, filesize=11.9 K 2024-11-21T00:16:20,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ec77d7e5be4bbf0794947ca9843b822c in 524ms, sequenceid=181, compaction requested=false 2024-11-21T00:16:20,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:20,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:16:20,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:20,365 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/afbe2382557d48b397484acb6a4f9d50 is 50, key is test_row_0/A:col10/1732148179790/Put/seqid=0 2024-11-21T00:16:20,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742097_1273 (size=12151) 2024-11-21T00:16:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:20,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148180455, max timestamp=1732148180521Num uniques 859; 2024-11-21T00:16:20,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148180455, max timestamp=1732148180521Num uniques 859; 2024-11-21T00:16:20,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:20,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:20,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148240547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148240552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148240557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148240557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148240558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148240669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148240669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148240681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148240684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148240688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,814 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/afbe2382557d48b397484acb6a4f9d50 2024-11-21T00:16:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/03ea3e71f82349fd88751241493175bb is 50, key is test_row_0/B:col10/1732148179790/Put/seqid=0 2024-11-21T00:16:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:20,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742098_1274 (size=12151) 2024-11-21T00:16:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148240896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148240896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148240899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148240899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148240908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148241208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148241216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148241216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148241216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148241227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,288 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/03ea3e71f82349fd88751241493175bb 2024-11-21T00:16:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/dfad404c2b5249ae9a150630377416aa is 50, key is test_row_0/C:col10/1732148179790/Put/seqid=0 2024-11-21T00:16:21,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742099_1275 (size=12151) 2024-11-21T00:16:21,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/dfad404c2b5249ae9a150630377416aa 2024-11-21T00:16:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/afbe2382557d48b397484acb6a4f9d50 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50 2024-11-21T00:16:21,716 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T00:16:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/03ea3e71f82349fd88751241493175bb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb 2024-11-21T00:16:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148241717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148241726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148241726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,734 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T00:16:21,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/dfad404c2b5249ae9a150630377416aa as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa 2024-11-21T00:16:21,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148241737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148241737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:21,744 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T00:16:21,745 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ec77d7e5be4bbf0794947ca9843b822c in 1381ms, sequenceid=195, compaction requested=true 2024-11-21T00:16:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T00:16:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T00:16:21,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T00:16:21,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-11-21T00:16:21,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0140 sec 2024-11-21T00:16:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:16:21,886 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:21,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T00:16:21,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:16:21,890 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:21,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:16:22,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:16:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:22,045 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=127.47 KB heapSize=363.78 KB 2024-11-21T00:16:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/eaac62fb71d648bc839a3ac93730b5ac is 50, key is test_row_0/A:col10/1732148180549/Put/seqid=0 2024-11-21T00:16:22,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742100_1276 (size=12151) 2024-11-21T00:16:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:16:22,493 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/eaac62fb71d648bc839a3ac93730b5ac 2024-11-21T00:16:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/2994c95f395845da9a06beb9b1e71150 is 50, key is test_row_0/B:col10/1732148180549/Put/seqid=0 2024-11-21T00:16:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:16:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742101_1277 (size=12151) 2024-11-21T00:16:22,550 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/2994c95f395845da9a06beb9b1e71150 2024-11-21T00:16:22,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a6a292389cb14e2f9d0e9aee70b41295 is 50, key is test_row_0/C:col10/1732148180549/Put/seqid=0 2024-11-21T00:16:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742102_1278 (size=12151) 2024-11-21T00:16:22,627 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a6a292389cb14e2f9d0e9aee70b41295 2024-11-21T00:16:22,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/eaac62fb71d648bc839a3ac93730b5ac as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac 2024-11-21T00:16:22,657 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac, entries=150, sequenceid=225, filesize=11.9 K 2024-11-21T00:16:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/2994c95f395845da9a06beb9b1e71150 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150 2024-11-21T00:16:22,671 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150, entries=150, sequenceid=225, filesize=11.9 K 2024-11-21T00:16:22,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a6a292389cb14e2f9d0e9aee70b41295 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295 2024-11-21T00:16:22,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295, entries=150, sequenceid=225, filesize=11.9 K 2024-11-21T00:16:22,691 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~363.73 KB/372464, currentSize=0 B/0 for ec77d7e5be4bbf0794947ca9843b822c in 646ms, sequenceid=225, compaction requested=true 2024-11-21T00:16:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T00:16:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T00:16:22,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T00:16:22,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 807 msec 2024-11-21T00:16:22,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 814 msec 2024-11-21T00:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:22,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:22,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/f1666227fb3b46e18194afac77540636 is 50, key is test_row_0/A:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:22,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742103_1279 (size=9757) 2024-11-21T00:16:22,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732148182825, max timestamp=1732148182840Num uniques 859; 2024-11-21T00:16:22,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:22,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148182825, max timestamp=1732148182840Num uniques 859; 2024-11-21T00:16:22,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732148182825, max timestamp=1732148182840Num uniques 859; 2024-11-21T00:16:22,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:22,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:22,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:22,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148242915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148242916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148242918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:22,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148242921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:22,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148242923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:16:23,030 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:23,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T00:16:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:23,032 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:23,034 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:23,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:23,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148243037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148243037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148243037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148243037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148243037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:23,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:16:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/f1666227fb3b46e18194afac77540636 2024-11-21T00:16:23,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148243246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148243246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148243246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148243246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/b17e36b9ac18459796258e0bcd450062 is 50, key is test_row_0/B:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148243256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742104_1280 (size=9757) 2024-11-21T00:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:23,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:16:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148243568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148243568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148243568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148243569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148243577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:23,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:16:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:23,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/b17e36b9ac18459796258e0bcd450062 2024-11-21T00:16:23,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/5aea29bd247b4bcaa3d6d7d38f1ba136 is 50, key is test_row_0/C:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742105_1281 (size=9757) 2024-11-21T00:16:23,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/5aea29bd247b4bcaa3d6d7d38f1ba136 2024-11-21T00:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/f1666227fb3b46e18194afac77540636 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636 2024-11-21T00:16:23,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636, entries=100, sequenceid=235, filesize=9.5 K 2024-11-21T00:16:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/b17e36b9ac18459796258e0bcd450062 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062 2024-11-21T00:16:23,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062, entries=100, sequenceid=235, filesize=9.5 K 2024-11-21T00:16:23,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/5aea29bd247b4bcaa3d6d7d38f1ba136 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136 2024-11-21T00:16:23,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136, entries=100, sequenceid=235, filesize=9.5 K 2024-11-21T00:16:23,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ec77d7e5be4bbf0794947ca9843b822c in 982ms, sequenceid=235, compaction requested=true 2024-11-21T00:16:23,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:23,764 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:16:23,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:23,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:23,764 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:16:23,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:23,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:23,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:23,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:23,771 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63483 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:16:23,771 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:23,771 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,771 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8ab927a75f8c41c2a3cbdd353fdf34cd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=62.0 K 2024-11-21T00:16:23,771 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58703 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:16:23,771 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:23,771 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,772 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9c8338c7b33f41f7aeba201857ae23da, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=57.3 K 2024-11-21T00:16:23,774 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab927a75f8c41c2a3cbdd353fdf34cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:23,774 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8338c7b33f41f7aeba201857ae23da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:23,775 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113cee5e598b48d3abc920a35fbf0ec2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148179119 2024-11-21T00:16:23,775 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c81efcb1cc349619ff7d4ce82bce657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148179119 2024-11-21T00:16:23,775 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting afbe2382557d48b397484acb6a4f9d50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148179784 2024-11-21T00:16:23,775 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ea3e71f82349fd88751241493175bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148179784 2024-11-21T00:16:23,775 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaac62fb71d648bc839a3ac93730b5ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732148180549 2024-11-21T00:16:23,776 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 2994c95f395845da9a06beb9b1e71150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732148180549 2024-11-21T00:16:23,776 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1666227fb3b46e18194afac77540636, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148182769 2024-11-21T00:16:23,777 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b17e36b9ac18459796258e0bcd450062, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148182769 2024-11-21T00:16:23,794 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#234 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:23,796 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:23,797 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/45ae8fe610d24ce897c8c7dc30d43e73 is 50, key is test_row_0/A:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:23,799 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/0526c60504ad4f8da85d57e831167ec8 is 50, key is test_row_0/B:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:23,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742106_1282 (size=12663) 2024-11-21T00:16:23,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:16:23,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:23,817 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9850f2b56fa342ea8b7d3a6e810fb0c6 is 50, key is test_row_0/A:col10/1732148182919/Put/seqid=0 2024-11-21T00:16:23,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742107_1283 (size=12663) 2024-11-21T00:16:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742108_1284 (size=12301) 2024-11-21T00:16:23,846 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9850f2b56fa342ea8b7d3a6e810fb0c6 2024-11-21T00:16:23,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9f051bdb8d19443fbfc6669e40e65204 is 50, key is test_row_0/B:col10/1732148182919/Put/seqid=0 2024-11-21T00:16:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742109_1285 (size=12301) 2024-11-21T00:16:23,890 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9f051bdb8d19443fbfc6669e40e65204 2024-11-21T00:16:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/15b098ef54f147fa822fef89dddaceb1 is 50, key is test_row_0/C:col10/1732148182919/Put/seqid=0 2024-11-21T00:16:23,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742110_1286 (size=12301) 2024-11-21T00:16:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148244098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148244100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148244104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148244110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148244111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148244216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148244216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148244216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,224 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/45ae8fe610d24ce897c8c7dc30d43e73 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/45ae8fe610d24ce897c8c7dc30d43e73 2024-11-21T00:16:24,230 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 45ae8fe610d24ce897c8c7dc30d43e73(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:24,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:24,230 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=11, startTime=1732148183764; duration=0sec 2024-11-21T00:16:24,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:24,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:24,230 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:16:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148244227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148244227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,233 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58703 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:16:24,233 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:24,234 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:24,234 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/645303fbe9a14589ba758db1e404ea07, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=57.3 K 2024-11-21T00:16:24,234 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 645303fbe9a14589ba758db1e404ea07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732148178404 2024-11-21T00:16:24,234 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867a64bf4935455fbcab598f7b16d07f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148179119 2024-11-21T00:16:24,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfad404c2b5249ae9a150630377416aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148179784 2024-11-21T00:16:24,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a292389cb14e2f9d0e9aee70b41295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732148180549 2024-11-21T00:16:24,235 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aea29bd247b4bcaa3d6d7d38f1ba136, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148182769 2024-11-21T00:16:24,254 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/0526c60504ad4f8da85d57e831167ec8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0526c60504ad4f8da85d57e831167ec8 2024-11-21T00:16:24,259 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into 0526c60504ad4f8da85d57e831167ec8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:24,259 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:24,259 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=11, startTime=1732148183764; duration=0sec 2024-11-21T00:16:24,259 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:24,259 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:24,261 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:24,261 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/94585dc107dc48999095da1991306bcd is 50, key is test_row_0/C:col10/1732148182777/Put/seqid=0 2024-11-21T00:16:24,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742111_1287 (size=12663) 2024-11-21T00:16:24,299 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/94585dc107dc48999095da1991306bcd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/94585dc107dc48999095da1991306bcd 2024-11-21T00:16:24,311 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 94585dc107dc48999095da1991306bcd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:24,311 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:24,312 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=11, startTime=1732148183767; duration=0sec 2024-11-21T00:16:24,312 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:24,312 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:24,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/15b098ef54f147fa822fef89dddaceb1 2024-11-21T00:16:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/9850f2b56fa342ea8b7d3a6e810fb0c6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6 2024-11-21T00:16:24,373 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6, entries=150, sequenceid=272, filesize=12.0 K 2024-11-21T00:16:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9f051bdb8d19443fbfc6669e40e65204 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204 2024-11-21T00:16:24,382 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204, entries=150, sequenceid=272, filesize=12.0 K 2024-11-21T00:16:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/15b098ef54f147fa822fef89dddaceb1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1 2024-11-21T00:16:24,388 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1, entries=150, sequenceid=272, filesize=12.0 K 2024-11-21T00:16:24,389 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for ec77d7e5be4bbf0794947ca9843b822c in 572ms, sequenceid=272, compaction requested=false 2024-11-21T00:16:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T00:16:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T00:16:24,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T00:16:24,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3570 sec 2024-11-21T00:16:24,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3640 sec 2024-11-21T00:16:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:24,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:24,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:24,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5da81ff772164346b20f0f6971e95b7f is 50, key is test_row_0/A:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:24,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148184456, max timestamp=1732148184467Num uniques 859; 2024-11-21T00:16:24,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:24,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148184456, max timestamp=1732148184467Num uniques 859; 2024-11-21T00:16:24,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148184456, max timestamp=1732148184467Num uniques 859; 2024-11-21T00:16:24,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:24,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:24,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742112_1288 (size=12301) 2024-11-21T00:16:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148244511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148244511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148244555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148244561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148244561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148244666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148244666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148244677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148244679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148244680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148244876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148244876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5da81ff772164346b20f0f6971e95b7f 2024-11-21T00:16:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148244886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148244886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148244887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:24,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/085f2d1aa17b4739a51248d0789afdd1 is 50, key is test_row_0/B:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:24,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742113_1289 (size=12301) 2024-11-21T00:16:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:16:25,179 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:25,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T00:16:25,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:25,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:25,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148245186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148245186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148245196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148245196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148245197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:25,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/085f2d1aa17b4739a51248d0789afdd1 2024-11-21T00:16:25,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/d31c5e64c16e4f4885487658b0b66d35 is 50, key is test_row_0/C:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:25,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:16:25,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:25,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742114_1290 (size=12301) 2024-11-21T00:16:25,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:16:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:25,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:25,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:16:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:25,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148245697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148245697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148245706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148245706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148245706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:25,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/d31c5e64c16e4f4885487658b0b66d35 2024-11-21T00:16:25,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5da81ff772164346b20f0f6971e95b7f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f 2024-11-21T00:16:25,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T00:16:25,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/085f2d1aa17b4739a51248d0789afdd1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1 2024-11-21T00:16:25,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T00:16:25,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/d31c5e64c16e4f4885487658b0b66d35 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35 2024-11-21T00:16:25,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T00:16:25,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ec77d7e5be4bbf0794947ca9843b822c in 1346ms, sequenceid=286, compaction requested=true 2024-11-21T00:16:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:25,776 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:16:25,777 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:25,780 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:25,780 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:25,780 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,780 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/45ae8fe610d24ce897c8c7dc30d43e73, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.4 K 2024-11-21T00:16:25,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:25,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:25,781 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,781 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0526c60504ad4f8da85d57e831167ec8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.4 K 2024-11-21T00:16:25,781 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45ae8fe610d24ce897c8c7dc30d43e73, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148180553 2024-11-21T00:16:25,782 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0526c60504ad4f8da85d57e831167ec8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148180553 2024-11-21T00:16:25,782 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9850f2b56fa342ea8b7d3a6e810fb0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732148182878 2024-11-21T00:16:25,782 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f051bdb8d19443fbfc6669e40e65204, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732148182878 2024-11-21T00:16:25,782 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da81ff772164346b20f0f6971e95b7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:25,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 085f2d1aa17b4739a51248d0789afdd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:25,801 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:25,802 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/e61247dc0f0f431ab5f1343e543de4b9 is 50, key is test_row_0/B:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:25,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,804 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T00:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:25,809 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:25,810 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/fc42204fd2d346eca643be3a0495206d is 50, key is test_row_0/A:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/2c3fe781352c4be085433ac5d3665404 is 50, key is test_row_0/A:col10/1732148184521/Put/seqid=0 2024-11-21T00:16:25,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742115_1291 (size=12915) 2024-11-21T00:16:25,859 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/e61247dc0f0f431ab5f1343e543de4b9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/e61247dc0f0f431ab5f1343e543de4b9 2024-11-21T00:16:25,866 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into e61247dc0f0f431ab5f1343e543de4b9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:25,866 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:25,866 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148185777; duration=0sec 2024-11-21T00:16:25,866 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:25,866 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:25,866 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:25,867 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:25,867 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:25,867 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:25,867 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/94585dc107dc48999095da1991306bcd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.4 K 2024-11-21T00:16:25,868 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 94585dc107dc48999095da1991306bcd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732148180553 2024-11-21T00:16:25,868 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b098ef54f147fa822fef89dddaceb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732148182878 2024-11-21T00:16:25,869 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d31c5e64c16e4f4885487658b0b66d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:25,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742116_1292 (size=12915) 2024-11-21T00:16:25,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742117_1293 (size=12301) 2024-11-21T00:16:25,882 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/2c3fe781352c4be085433ac5d3665404 2024-11-21T00:16:25,899 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:25,899 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/417a19f2ff7b40b681b1e6074a6a66dd is 50, key is test_row_0/C:col10/1732148184429/Put/seqid=0 2024-11-21T00:16:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/90bdb1fcaf9146dcba11e3185c08327d is 50, key is test_row_0/B:col10/1732148184521/Put/seqid=0 2024-11-21T00:16:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742118_1294 (size=12915) 2024-11-21T00:16:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742119_1295 (size=12301) 2024-11-21T00:16:26,301 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/fc42204fd2d346eca643be3a0495206d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/fc42204fd2d346eca643be3a0495206d 2024-11-21T00:16:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:26,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/90bdb1fcaf9146dcba11e3185c08327d 2024-11-21T00:16:26,336 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into fc42204fd2d346eca643be3a0495206d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:26,337 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:26,337 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148185776; duration=0sec 2024-11-21T00:16:26,338 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:26,338 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:26,338 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/417a19f2ff7b40b681b1e6074a6a66dd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/417a19f2ff7b40b681b1e6074a6a66dd 2024-11-21T00:16:26,344 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 417a19f2ff7b40b681b1e6074a6a66dd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:26,344 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:26,344 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148185777; duration=0sec 2024-11-21T00:16:26,344 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:26,344 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/f527b64846bc4925988b45ddb04614c4 is 50, key is test_row_0/C:col10/1732148184521/Put/seqid=0 2024-11-21T00:16:26,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742120_1296 (size=12301) 2024-11-21T00:16:26,405 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/f527b64846bc4925988b45ddb04614c4 2024-11-21T00:16:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/2c3fe781352c4be085433ac5d3665404 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404 2024-11-21T00:16:26,455 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T00:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/90bdb1fcaf9146dcba11e3185c08327d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d 2024-11-21T00:16:26,461 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T00:16:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/f527b64846bc4925988b45ddb04614c4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4 2024-11-21T00:16:26,474 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T00:16:26,480 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for ec77d7e5be4bbf0794947ca9843b822c in 676ms, sequenceid=321, compaction requested=false 2024-11-21T00:16:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T00:16:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T00:16:26,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T00:16:26,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-21T00:16:26,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3110 sec 2024-11-21T00:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:26,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T00:16:26,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:26,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:26,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:26,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/8268e23ef8cc44dca800a526646bdc20 is 50, key is test_row_0/A:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742121_1297 (size=12301) 2024-11-21T00:16:26,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/8268e23ef8cc44dca800a526646bdc20 2024-11-21T00:16:26,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/85fa0b3d948e4181bb47ea71333ed5d3 is 50, key is test_row_0/B:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:26,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742122_1298 (size=12301) 2024-11-21T00:16:26,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732148186771, max timestamp=1732148186830Num uniques 859; 2024-11-21T00:16:26,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:26,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148186771, max timestamp=1732148186830Num uniques 859; 2024-11-21T00:16:26,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:16:26,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148186771, max timestamp=1732148186830Num uniques 859; 2024-11-21T00:16:26,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:16:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148246897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148246900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148246900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148246901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:26,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148246902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148247021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148247021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148247021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148247020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148247024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/85fa0b3d948e4181bb47ea71333ed5d3 2024-11-21T00:16:27,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/7cbfcb5ebc9f42b3b5f734302fee23c2 is 50, key is test_row_0/C:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:27,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148247238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148247238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148247239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148247248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148247249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742123_1299 (size=12301) 2024-11-21T00:16:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:16:27,336 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:27,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T00:16:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:27,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:27,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:27,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:27,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T00:16:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148247560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148247561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148247561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148247569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148247569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:27,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/7cbfcb5ebc9f42b3b5f734302fee23c2 2024-11-21T00:16:27,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T00:16:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/8268e23ef8cc44dca800a526646bdc20 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20 2024-11-21T00:16:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:27,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T00:16:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/85fa0b3d948e4181bb47ea71333ed5d3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3 2024-11-21T00:16:27,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T00:16:27,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/7cbfcb5ebc9f42b3b5f734302fee23c2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2 2024-11-21T00:16:27,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T00:16:27,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=128.92 KB/132012 for ec77d7e5be4bbf0794947ca9843b822c in 985ms, sequenceid=336, compaction requested=true 2024-11-21T00:16:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:27,737 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:16:27,737 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:27,748 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:27,748 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:27,748 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:27,748 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,748 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:27,748 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/fc42204fd2d346eca643be3a0495206d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.6 K 2024-11-21T00:16:27,748 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,748 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/e61247dc0f0f431ab5f1343e543de4b9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.6 K 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e61247dc0f0f431ab5f1343e543de4b9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc42204fd2d346eca643be3a0495206d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bdb1fcaf9146dcba11e3185c08327d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148184473 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c3fe781352c4be085433ac5d3665404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148184473 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 85fa0b3d948e4181bb47ea71333ed5d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:27,753 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8268e23ef8cc44dca800a526646bdc20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:27,770 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:27,771 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/f3759d0bf81e487ab5778eccb832b91c is 50, key is test_row_0/B:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:27,778 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:27,779 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/c0dce7a0593c4945ab46cc716f4efb3e is 50, key is test_row_0/A:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742125_1301 (size=13017) 2024-11-21T00:16:27,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742124_1300 (size=13017) 2024-11-21T00:16:27,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T00:16:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:27,852 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=128.92 KB heapSize=368.31 KB 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:27,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/1278a754c6f5417d9ceed835f48944b0 is 50, key is test_row_0/A:col10/1732148186896/Put/seqid=0 2024-11-21T00:16:27,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742126_1302 (size=12301) 2024-11-21T00:16:27,935 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/1278a754c6f5417d9ceed835f48944b0 2024-11-21T00:16:27,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/a03252ab3e604942b2c3de23c5f34dfa is 50, key is test_row_0/B:col10/1732148186896/Put/seqid=0 2024-11-21T00:16:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:28,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742127_1303 (size=12301) 2024-11-21T00:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:28,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:28,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148248121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148248124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148248133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148248133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148248134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148248236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148248248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148248248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148248248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148248248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,260 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/c0dce7a0593c4945ab46cc716f4efb3e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c0dce7a0593c4945ab46cc716f4efb3e 2024-11-21T00:16:28,271 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/f3759d0bf81e487ab5778eccb832b91c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f3759d0bf81e487ab5778eccb832b91c 2024-11-21T00:16:28,288 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into c0dce7a0593c4945ab46cc716f4efb3e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:28,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:28,288 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148187736; duration=0sec 2024-11-21T00:16:28,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:28,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:28,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:28,307 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into f3759d0bf81e487ab5778eccb832b91c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:28,307 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:28,307 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148187737; duration=0sec 2024-11-21T00:16:28,307 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:28,307 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:28,308 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:28,308 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:28,308 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:28,308 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/417a19f2ff7b40b681b1e6074a6a66dd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.6 K 2024-11-21T00:16:28,316 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417a19f2ff7b40b681b1e6074a6a66dd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732148184107 2024-11-21T00:16:28,321 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f527b64846bc4925988b45ddb04614c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148184473 2024-11-21T00:16:28,324 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cbfcb5ebc9f42b3b5f734302fee23c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:28,418 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:28,419 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6b29b6de426a4ca89230d201c5924321 is 50, key is test_row_0/C:col10/1732148186736/Put/seqid=0 2024-11-21T00:16:28,432 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/a03252ab3e604942b2c3de23c5f34dfa 2024-11-21T00:16:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148248448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148248457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/1e146ca950364917b1831eba25c0a8a9 is 50, key is test_row_0/C:col10/1732148186896/Put/seqid=0 2024-11-21T00:16:28,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742128_1304 (size=13017) 2024-11-21T00:16:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148248466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148248466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148248466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742129_1305 (size=12301) 2024-11-21T00:16:28,497 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/1e146ca950364917b1831eba25c0a8a9 2024-11-21T00:16:28,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/1278a754c6f5417d9ceed835f48944b0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0 2024-11-21T00:16:28,556 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T00:16:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/a03252ab3e604942b2c3de23c5f34dfa as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa 2024-11-21T00:16:28,572 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T00:16:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/1e146ca950364917b1831eba25c0a8a9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9 2024-11-21T00:16:28,634 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T00:16:28,635 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~128.92 KB/132012, heapSize ~368.27 KB/377104, currentSize=60.38 KB/61830 for ec77d7e5be4bbf0794947ca9843b822c in 783ms, sequenceid=372, compaction requested=false 2024-11-21T00:16:28,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T00:16:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T00:16:28,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T00:16:28,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2910 sec 2024-11-21T00:16:28,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3240 sec 2024-11-21T00:16:28,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:16:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:28,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/16e4d91bc6e640e1b366d3e499696be9 is 50, key is test_row_0/A:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:28,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742130_1306 (size=12301) 2024-11-21T00:16:28,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/16e4d91bc6e640e1b366d3e499696be9 2024-11-21T00:16:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148248833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148248838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148248843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148248845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148248845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/5f1783fade1d48a6970dbcc123c1a78a is 50, key is test_row_0/B:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:28,926 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/6b29b6de426a4ca89230d201c5924321 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6b29b6de426a4ca89230d201c5924321 2024-11-21T00:16:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742131_1307 (size=12301) 2024-11-21T00:16:28,937 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into 6b29b6de426a4ca89230d201c5924321(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:28,937 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:28,937 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148187737; duration=0sec 2024-11-21T00:16:28,937 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:28,937 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148248957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148248957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148248960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148248964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:28,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148248972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148249167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148249167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148249177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148249177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148249188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/5f1783fade1d48a6970dbcc123c1a78a 2024-11-21T00:16:29,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b85b931565647539af4bf0e7b363dde is 50, key is test_row_0/C:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:29,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742132_1308 (size=12301) 2024-11-21T00:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:16:29,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:29,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T00:16:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:29,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:29,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:29,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:29,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148249487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148249489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148249497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148249497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148249497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:29,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T00:16:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T00:16:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:29,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b85b931565647539af4bf0e7b363dde 2024-11-21T00:16:29,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/16e4d91bc6e640e1b366d3e499696be9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9 2024-11-21T00:16:29,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T00:16:29,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/5f1783fade1d48a6970dbcc123c1a78a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a 2024-11-21T00:16:29,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T00:16:29,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/2b85b931565647539af4bf0e7b363dde as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde 2024-11-21T00:16:29,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T00:16:29,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ec77d7e5be4bbf0794947ca9843b822c in 1130ms, sequenceid=388, compaction requested=true 2024-11-21T00:16:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:29,909 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:29,909 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:29,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:29,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:29,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:29,917 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,917 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f3759d0bf81e487ab5778eccb832b91c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.7 K 2024-11-21T00:16:29,918 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:29,918 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:29,918 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,918 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c0dce7a0593c4945ab46cc716f4efb3e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.7 K 2024-11-21T00:16:29,918 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f3759d0bf81e487ab5778eccb832b91c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:29,919 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0dce7a0593c4945ab46cc716f4efb3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:29,919 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a03252ab3e604942b2c3de23c5f34dfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732148186896 2024-11-21T00:16:29,919 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1278a754c6f5417d9ceed835f48944b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732148186896 2024-11-21T00:16:29,919 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1783fade1d48a6970dbcc123c1a78a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:29,920 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e4d91bc6e640e1b366d3e499696be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:29,940 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:29,940 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/efa00d98e6354857841e0656becc642e is 50, key is test_row_0/B:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:29,951 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:29,951 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/6653ad58d18040ce9fbd2529d68cfa9d is 50, key is test_row_0/A:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:29,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T00:16:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:29,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/367578fcae864587b9d4b148a3fa35ae is 50, key is test_row_0/A:col10/1732148188843/Put/seqid=0 2024-11-21T00:16:29,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742133_1309 (size=13119) 2024-11-21T00:16:30,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742134_1310 (size=13119) 2024-11-21T00:16:30,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:30,017 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/efa00d98e6354857841e0656becc642e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/efa00d98e6354857841e0656becc642e 2024-11-21T00:16:30,017 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/6653ad58d18040ce9fbd2529d68cfa9d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/6653ad58d18040ce9fbd2529d68cfa9d 2024-11-21T00:16:30,023 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into efa00d98e6354857841e0656becc642e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:30,023 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:30,023 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148189909; duration=0sec 2024-11-21T00:16:30,023 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:30,023 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:30,023 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:30,025 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:30,025 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:30,025 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:30,025 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6b29b6de426a4ca89230d201c5924321, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.7 K 2024-11-21T00:16:30,025 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b29b6de426a4ca89230d201c5924321, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732148186710 2024-11-21T00:16:30,026 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e146ca950364917b1831eba25c0a8a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732148186896 2024-11-21T00:16:30,026 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b85b931565647539af4bf0e7b363dde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:30,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742135_1311 (size=12301) 2024-11-21T00:16:30,037 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:30,038 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 6653ad58d18040ce9fbd2529d68cfa9d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:30,038 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:30,038 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148189909; duration=0sec 2024-11-21T00:16:30,038 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:30,038 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:30,038 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a32e2fa9197d4b75a682d8aa4f70aa60 is 50, key is test_row_0/C:col10/1732148188124/Put/seqid=0 2024-11-21T00:16:30,045 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/367578fcae864587b9d4b148a3fa35ae 2024-11-21T00:16:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742136_1312 (size=13119) 2024-11-21T00:16:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/f8993dd7ec044ac799b3bb45e3d57eb9 is 50, key is test_row_0/B:col10/1732148188843/Put/seqid=0 2024-11-21T00:16:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148250053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148250052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148250074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148250074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148250074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:30,117 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/a32e2fa9197d4b75a682d8aa4f70aa60 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a32e2fa9197d4b75a682d8aa4f70aa60 2024-11-21T00:16:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742137_1313 (size=12301) 2024-11-21T00:16:30,124 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/f8993dd7ec044ac799b3bb45e3d57eb9 2024-11-21T00:16:30,167 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into a32e2fa9197d4b75a682d8aa4f70aa60(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:30,167 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:30,167 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148189912; duration=0sec 2024-11-21T00:16:30,167 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:30,167 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:30,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/4dbd9b462fa645adb8986dc37d53335c is 50, key is test_row_0/C:col10/1732148188843/Put/seqid=0 2024-11-21T00:16:30,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148250180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148250180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148250189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148250189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148250196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742138_1314 (size=12301) 2024-11-21T00:16:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148250396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148250396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148250396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148250396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148250407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,604 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/4dbd9b462fa645adb8986dc37d53335c 2024-11-21T00:16:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/367578fcae864587b9d4b148a3fa35ae as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae 2024-11-21T00:16:30,661 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:16:30,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/f8993dd7ec044ac799b3bb45e3d57eb9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9 2024-11-21T00:16:30,669 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:16:30,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/4dbd9b462fa645adb8986dc37d53335c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c 2024-11-21T00:16:30,683 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:16:30,685 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ec77d7e5be4bbf0794947ca9843b822c in 727ms, sequenceid=408, compaction requested=false 2024-11-21T00:16:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T00:16:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T00:16:30,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T00:16:30,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-21T00:16:30,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.2100 sec 2024-11-21T00:16:30,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:30,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/452f78be54304203b84c9b06539957be is 50, key is test_row_0/A:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:30,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742139_1315 (size=12301) 2024-11-21T00:16:30,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/452f78be54304203b84c9b06539957be 2024-11-21T00:16:30,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148250743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/c99943d89b5e4f03bf5805d11ddf7f57 is 50, key is test_row_0/B:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148250748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148250751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148250760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148250761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742140_1316 (size=12301) 2024-11-21T00:16:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148250866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148250866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148250866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148250866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148250877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148251076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148251077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148251077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148251080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148251087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/c99943d89b5e4f03bf5805d11ddf7f57 2024-11-21T00:16:31,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/32915d9b82ee43ac8132694c73690b1c is 50, key is test_row_0/C:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742141_1317 (size=12301) 2024-11-21T00:16:31,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/32915d9b82ee43ac8132694c73690b1c 2024-11-21T00:16:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/452f78be54304203b84c9b06539957be as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be 2024-11-21T00:16:31,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T00:16:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/c99943d89b5e4f03bf5805d11ddf7f57 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57 2024-11-21T00:16:31,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T00:16:31,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/32915d9b82ee43ac8132694c73690b1c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c 2024-11-21T00:16:31,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T00:16:31,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ec77d7e5be4bbf0794947ca9843b822c in 565ms, sequenceid=426, compaction requested=true 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:31,285 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:31,285 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec77d7e5be4bbf0794947ca9843b822c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:31,286 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:31,286 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/B is initiating minor compaction (all files) 2024-11-21T00:16:31,287 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/B in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,287 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/efa00d98e6354857841e0656becc642e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.8 K 2024-11-21T00:16:31,287 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting efa00d98e6354857841e0656becc642e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:31,288 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f8993dd7ec044ac799b3bb45e3d57eb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148188827 2024-11-21T00:16:31,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:31,288 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c99943d89b5e4f03bf5805d11ddf7f57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732148190039 2024-11-21T00:16:31,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/A is initiating minor compaction (all files) 2024-11-21T00:16:31,288 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/A in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,288 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/6653ad58d18040ce9fbd2529d68cfa9d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.8 K 2024-11-21T00:16:31,288 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6653ad58d18040ce9fbd2529d68cfa9d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:31,289 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 367578fcae864587b9d4b148a3fa35ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148188827 2024-11-21T00:16:31,289 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452f78be54304203b84c9b06539957be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732148190039 2024-11-21T00:16:31,299 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:31,299 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/d48cbe6df2b047efaaeca255cd680f66 is 50, key is test_row_0/B:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:31,319 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:31,320 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/768c70334bd14f3b99b04a24e32d6b12 is 50, key is test_row_0/A:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:31,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742143_1319 (size=13221) 2024-11-21T00:16:31,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742142_1318 (size=13221) 2024-11-21T00:16:31,365 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/768c70334bd14f3b99b04a24e32d6b12 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/768c70334bd14f3b99b04a24e32d6b12 2024-11-21T00:16:31,374 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/A of ec77d7e5be4bbf0794947ca9843b822c into 768c70334bd14f3b99b04a24e32d6b12(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:31,374 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:31,374 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/A, priority=13, startTime=1732148191285; duration=0sec 2024-11-21T00:16:31,376 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:31,376 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:A 2024-11-21T00:16:31,376 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:31,378 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:31,378 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): ec77d7e5be4bbf0794947ca9843b822c/C is initiating minor compaction (all files) 2024-11-21T00:16:31,378 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec77d7e5be4bbf0794947ca9843b822c/C in TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,379 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a32e2fa9197d4b75a682d8aa4f70aa60, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp, totalSize=36.8 K 2024-11-21T00:16:31,379 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32e2fa9197d4b75a682d8aa4f70aa60, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732148188124 2024-11-21T00:16:31,379 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dbd9b462fa645adb8986dc37d53335c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148188827 2024-11-21T00:16:31,379 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32915d9b82ee43ac8132694c73690b1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732148190039 2024-11-21T00:16:31,396 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec77d7e5be4bbf0794947ca9843b822c#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:31,398 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/c79b800b06e2423a9929d3b714eabf0c is 50, key is test_row_0/C:col10/1732148190062/Put/seqid=0 2024-11-21T00:16:31,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:16:31,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:31,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:31,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:31,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5b4e7219f60e4f2a9f60a311e74b26b1 is 50, key is test_row_0/A:col10/1732148191404/Put/seqid=0 2024-11-21T00:16:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148251432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742144_1320 (size=13221) 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148251434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148251434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148251438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148251445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,460 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/c79b800b06e2423a9929d3b714eabf0c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/c79b800b06e2423a9929d3b714eabf0c 2024-11-21T00:16:31,467 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/C of ec77d7e5be4bbf0794947ca9843b822c into c79b800b06e2423a9929d3b714eabf0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:31,467 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:31,467 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/C, priority=13, startTime=1732148191285; duration=0sec 2024-11-21T00:16:31,468 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:31,468 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:C 2024-11-21T00:16:31,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742145_1321 (size=12301) 2024-11-21T00:16:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148251553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148251557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148251557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148251557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148251557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T00:16:31,636 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:31,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T00:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:31,639 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:31,640 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:31,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:31,753 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/d48cbe6df2b047efaaeca255cd680f66 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d48cbe6df2b047efaaeca255cd680f66 2024-11-21T00:16:31,760 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec77d7e5be4bbf0794947ca9843b822c/B of ec77d7e5be4bbf0794947ca9843b822c into d48cbe6df2b047efaaeca255cd680f66(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:31,760 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:31,760 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., storeName=ec77d7e5be4bbf0794947ca9843b822c/B, priority=13, startTime=1732148191285; duration=0sec 2024-11-21T00:16:31,760 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:31,760 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec77d7e5be4bbf0794947ca9843b822c:B 2024-11-21T00:16:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148251766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148251766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148251766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148251767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148251767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:31,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5b4e7219f60e4f2a9f60a311e74b26b1 2024-11-21T00:16:31,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9d05bf68f1b7424abb99dd6e4d297f00 is 50, key is test_row_0/B:col10/1732148191404/Put/seqid=0 2024-11-21T00:16:31,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742146_1322 (size=12301) 2024-11-21T00:16:31,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:31,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148252076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148252076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148252076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148252076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148252088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:32,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:32,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:32,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9d05bf68f1b7424abb99dd6e4d297f00 2024-11-21T00:16:32,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/db0a1d38e1c447228285343a449bf2aa is 50, key is test_row_0/C:col10/1732148191404/Put/seqid=0 2024-11-21T00:16:32,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742147_1323 (size=12301) 2024-11-21T00:16:32,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:32,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50114 deadline: 1732148252586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50140 deadline: 1732148252587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50158 deadline: 1732148252587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50154 deadline: 1732148252587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50088 deadline: 1732148252608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:32,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:16:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/db0a1d38e1c447228285343a449bf2aa 2024-11-21T00:16:32,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/5b4e7219f60e4f2a9f60a311e74b26b1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5b4e7219f60e4f2a9f60a311e74b26b1 2024-11-21T00:16:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:32,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5b4e7219f60e4f2a9f60a311e74b26b1, entries=150, sequenceid=445, filesize=12.0 K 2024-11-21T00:16:32,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/9d05bf68f1b7424abb99dd6e4d297f00 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9d05bf68f1b7424abb99dd6e4d297f00 2024-11-21T00:16:32,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9d05bf68f1b7424abb99dd6e4d297f00, entries=150, sequenceid=445, filesize=12.0 K 2024-11-21T00:16:32,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/db0a1d38e1c447228285343a449bf2aa as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/db0a1d38e1c447228285343a449bf2aa 2024-11-21T00:16:32,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/db0a1d38e1c447228285343a449bf2aa, entries=150, sequenceid=445, filesize=12.0 K 2024-11-21T00:16:32,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ec77d7e5be4bbf0794947ca9843b822c in 1410ms, sequenceid=445, compaction requested=false 2024-11-21T00:16:32,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:32,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T00:16:32,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:32,937 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:32,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/d403a45411bf42b4aac4e4ba07998d96 is 50, key is test_row_0/A:col10/1732148191435/Put/seqid=0 2024-11-21T00:16:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742148_1324 (size=12301) 2024-11-21T00:16:33,105 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-21T00:16:33,105 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,105 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,106 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,109 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-21T00:16:33,109 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,109 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,110 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-21T00:16:33,110 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,110 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,110 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,110 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,111 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-21T00:16:33,111 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,111 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,111 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,112 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-21T00:16:33,113 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,113 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,113 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,453 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/d403a45411bf42b4aac4e4ba07998d96 2024-11-21T00:16:33,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1722dce54cfc4b6f915c658dcedf59f2 is 50, key is test_row_0/B:col10/1732148191435/Put/seqid=0 2024-11-21T00:16:33,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742149_1325 (size=12301) 2024-11-21T00:16:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:33,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. as already flushing 2024-11-21T00:16:33,598 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-21T00:16:33,598 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-21T00:16:33,598 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,598 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-21T00:16:33,598 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,599 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,599 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,600 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-21T00:16:33,600 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,600 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,600 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,629 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-21T00:16:33,629 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:16:33,629 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,629 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:33,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:33,889 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1722dce54cfc4b6f915c658dcedf59f2 2024-11-21T00:16:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/58ba6a3e3d8d4920ae97f3910d704ee5 is 50, key is test_row_0/C:col10/1732148191435/Put/seqid=0 2024-11-21T00:16:33,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742150_1326 (size=12301) 2024-11-21T00:16:34,302 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/58ba6a3e3d8d4920ae97f3910d704ee5 2024-11-21T00:16:34,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/d403a45411bf42b4aac4e4ba07998d96 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/d403a45411bf42b4aac4e4ba07998d96 2024-11-21T00:16:34,326 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/d403a45411bf42b4aac4e4ba07998d96, entries=150, sequenceid=461, filesize=12.0 K 2024-11-21T00:16:34,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/1722dce54cfc4b6f915c658dcedf59f2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1722dce54cfc4b6f915c658dcedf59f2 2024-11-21T00:16:34,332 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1722dce54cfc4b6f915c658dcedf59f2, entries=150, sequenceid=461, filesize=12.0 K 2024-11-21T00:16:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/58ba6a3e3d8d4920ae97f3910d704ee5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/58ba6a3e3d8d4920ae97f3910d704ee5 2024-11-21T00:16:34,338 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/58ba6a3e3d8d4920ae97f3910d704ee5, entries=150, sequenceid=461, filesize=12.0 K 2024-11-21T00:16:34,339 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for ec77d7e5be4bbf0794947ca9843b822c in 1402ms, sequenceid=461, compaction requested=true 2024-11-21T00:16:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ec77d7e5be4bbf0794947ca9843b822c: 2024-11-21T00:16:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T00:16:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T00:16:34,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T00:16:34,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7160 sec 2024-11-21T00:16:34,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.7290 sec 2024-11-21T00:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:16:35,817 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-21T00:16:35,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1956 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1916 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1882 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1922 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1933 2024-11-21T00:16:35,818 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:16:35,818 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:16:35,818 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:16:35,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:35,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:35,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:16:35,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:35,824 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148195824"}]},"ts":"1732148195824"} 2024-11-21T00:16:35,826 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:16:35,826 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:16:35,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:16:35,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, UNASSIGN}] 2024-11-21T00:16:35,828 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, UNASSIGN 2024-11-21T00:16:35,829 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ec77d7e5be4bbf0794947ca9843b822c, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:35,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, UNASSIGN because future has completed 2024-11-21T00:16:35,831 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:16:35,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:35,983 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:35,983 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:16:35,983 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing ec77d7e5be4bbf0794947ca9843b822c, disabling compactions & flushes 2024-11-21T00:16:35,983 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:35,983 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:35,983 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. after waiting 0 ms 2024-11-21T00:16:35,983 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:35,983 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing ec77d7e5be4bbf0794947ca9843b822c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=A 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=B 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec77d7e5be4bbf0794947ca9843b822c, store=C 2024-11-21T00:16:35,984 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:35,987 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e832ed4683244e80af84e361cd616577 is 50, key is test_row_1/A:col10/1732148193627/Put/seqid=0 2024-11-21T00:16:35,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742151_1327 (size=9857) 2024-11-21T00:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:36,396 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e832ed4683244e80af84e361cd616577 2024-11-21T00:16:36,403 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/8045936671234c6e9d5a7765a29cfda9 is 50, key is test_row_1/B:col10/1732148193627/Put/seqid=0 2024-11-21T00:16:36,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742152_1328 (size=9857) 2024-11-21T00:16:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:36,808 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/8045936671234c6e9d5a7765a29cfda9 2024-11-21T00:16:36,814 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/3fc7ea1872a648ed94ce4bb151ddb2a0 is 50, key is test_row_1/C:col10/1732148193627/Put/seqid=0 2024-11-21T00:16:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742153_1329 (size=9857) 2024-11-21T00:16:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:37,217 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/3fc7ea1872a648ed94ce4bb151ddb2a0 2024-11-21T00:16:37,222 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/A/e832ed4683244e80af84e361cd616577 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e832ed4683244e80af84e361cd616577 2024-11-21T00:16:37,227 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e832ed4683244e80af84e361cd616577, entries=100, sequenceid=469, filesize=9.6 K 2024-11-21T00:16:37,227 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/B/8045936671234c6e9d5a7765a29cfda9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/8045936671234c6e9d5a7765a29cfda9 2024-11-21T00:16:37,231 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/8045936671234c6e9d5a7765a29cfda9, entries=100, sequenceid=469, filesize=9.6 K 2024-11-21T00:16:37,231 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/.tmp/C/3fc7ea1872a648ed94ce4bb151ddb2a0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/3fc7ea1872a648ed94ce4bb151ddb2a0 2024-11-21T00:16:37,236 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/3fc7ea1872a648ed94ce4bb151ddb2a0, entries=100, sequenceid=469, filesize=9.6 K 2024-11-21T00:16:37,237 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ec77d7e5be4bbf0794947ca9843b822c in 1253ms, sequenceid=469, compaction requested=true 2024-11-21T00:16:37,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/7f5b6864af8b458c9b45efe866d68bf2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9185018f03ec4304880a1c36d3b7e350, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8ab927a75f8c41c2a3cbdd353fdf34cd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/45ae8fe610d24ce897c8c7dc30d43e73, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/fc42204fd2d346eca643be3a0495206d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c0dce7a0593c4945ab46cc716f4efb3e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/6653ad58d18040ce9fbd2529d68cfa9d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be] to archive 2024-11-21T00:16:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:37,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/762f3c538ff14c6a87957f3a35fcb6ca 2024-11-21T00:16:37,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e3e8b3d192a44eab96ef36ae7cb37edd 2024-11-21T00:16:37,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9ac8405d98f04c6eb94b02d24742cdb1 2024-11-21T00:16:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/7f5b6864af8b458c9b45efe866d68bf2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/7f5b6864af8b458c9b45efe866d68bf2 2024-11-21T00:16:37,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/28badd63158e4b829ae2ffd527e8c9f4 2024-11-21T00:16:37,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/cc4d7c1c346347e2a31d8ef531744a88 2024-11-21T00:16:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5ed89db690f3431b84f86ccbe7234e0d 2024-11-21T00:16:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9185018f03ec4304880a1c36d3b7e350 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9185018f03ec4304880a1c36d3b7e350 2024-11-21T00:16:37,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c2a6281f306040a39c6fb7556abd7747 2024-11-21T00:16:37,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8ab927a75f8c41c2a3cbdd353fdf34cd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8ab927a75f8c41c2a3cbdd353fdf34cd 2024-11-21T00:16:37,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5f92f5bec52645c890d18998e51903de 2024-11-21T00:16:37,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/113cee5e598b48d3abc920a35fbf0ec2 2024-11-21T00:16:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/afbe2382557d48b397484acb6a4f9d50 2024-11-21T00:16:37,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/eaac62fb71d648bc839a3ac93730b5ac 2024-11-21T00:16:37,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/45ae8fe610d24ce897c8c7dc30d43e73 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/45ae8fe610d24ce897c8c7dc30d43e73 2024-11-21T00:16:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/f1666227fb3b46e18194afac77540636 2024-11-21T00:16:37,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/9850f2b56fa342ea8b7d3a6e810fb0c6 2024-11-21T00:16:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/fc42204fd2d346eca643be3a0495206d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/fc42204fd2d346eca643be3a0495206d 2024-11-21T00:16:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5da81ff772164346b20f0f6971e95b7f 2024-11-21T00:16:37,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/2c3fe781352c4be085433ac5d3665404 2024-11-21T00:16:37,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c0dce7a0593c4945ab46cc716f4efb3e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/c0dce7a0593c4945ab46cc716f4efb3e 2024-11-21T00:16:37,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/8268e23ef8cc44dca800a526646bdc20 2024-11-21T00:16:37,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/1278a754c6f5417d9ceed835f48944b0 2024-11-21T00:16:37,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/6653ad58d18040ce9fbd2529d68cfa9d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/6653ad58d18040ce9fbd2529d68cfa9d 2024-11-21T00:16:37,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/16e4d91bc6e640e1b366d3e499696be9 2024-11-21T00:16:37,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/367578fcae864587b9d4b148a3fa35ae 2024-11-21T00:16:37,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/452f78be54304203b84c9b06539957be 2024-11-21T00:16:37,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2b9077aaa9b84ef1926b692849e1b282, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/78c2fee9b78142a2b3af4b0665fd4d95, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9c8338c7b33f41f7aeba201857ae23da, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0526c60504ad4f8da85d57e831167ec8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/e61247dc0f0f431ab5f1343e543de4b9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f3759d0bf81e487ab5778eccb832b91c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/efa00d98e6354857841e0656becc642e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57] to archive 2024-11-21T00:16:37,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:37,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0c5e37aba9b24b4d8082ee4460baf86b 2024-11-21T00:16:37,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/fdc474587f124a4eb59918dc1c9c3507 2024-11-21T00:16:37,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d6c2088174fb496496efe14ad04207f1 2024-11-21T00:16:37,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2b9077aaa9b84ef1926b692849e1b282 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2b9077aaa9b84ef1926b692849e1b282 2024-11-21T00:16:37,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/ac70468516474864ac1999524db876ab 2024-11-21T00:16:37,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/add79d74e99d478393bbcc9a4424a942 2024-11-21T00:16:37,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/78c2fee9b78142a2b3af4b0665fd4d95 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/78c2fee9b78142a2b3af4b0665fd4d95 2024-11-21T00:16:37,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/3938561a199b41ebb3c5e2373cd71cb0 2024-11-21T00:16:37,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/179077e9f2ac4187b01a22b637db113e 2024-11-21T00:16:37,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9c8338c7b33f41f7aeba201857ae23da to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9c8338c7b33f41f7aeba201857ae23da 2024-11-21T00:16:37,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/6cc9d7d52ad64cf4b1b7728c18950bcb 2024-11-21T00:16:37,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1c81efcb1cc349619ff7d4ce82bce657 2024-11-21T00:16:37,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/03ea3e71f82349fd88751241493175bb 2024-11-21T00:16:37,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/2994c95f395845da9a06beb9b1e71150 2024-11-21T00:16:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0526c60504ad4f8da85d57e831167ec8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/0526c60504ad4f8da85d57e831167ec8 2024-11-21T00:16:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/b17e36b9ac18459796258e0bcd450062 2024-11-21T00:16:37,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9f051bdb8d19443fbfc6669e40e65204 2024-11-21T00:16:37,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/e61247dc0f0f431ab5f1343e543de4b9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/e61247dc0f0f431ab5f1343e543de4b9 2024-11-21T00:16:37,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/085f2d1aa17b4739a51248d0789afdd1 2024-11-21T00:16:37,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/90bdb1fcaf9146dcba11e3185c08327d 2024-11-21T00:16:37,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f3759d0bf81e487ab5778eccb832b91c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f3759d0bf81e487ab5778eccb832b91c 2024-11-21T00:16:37,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/85fa0b3d948e4181bb47ea71333ed5d3 2024-11-21T00:16:37,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/a03252ab3e604942b2c3de23c5f34dfa 2024-11-21T00:16:37,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/efa00d98e6354857841e0656becc642e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/efa00d98e6354857841e0656becc642e 2024-11-21T00:16:37,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/5f1783fade1d48a6970dbcc123c1a78a 2024-11-21T00:16:37,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/f8993dd7ec044ac799b3bb45e3d57eb9 2024-11-21T00:16:37,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/c99943d89b5e4f03bf5805d11ddf7f57 2024-11-21T00:16:37,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/019e0223f95f4a27b9c8b16c0c50fdbd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2e21c4d53f2947ad9bd11bd0f6774a0b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/645303fbe9a14589ba758db1e404ea07, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/94585dc107dc48999095da1991306bcd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/417a19f2ff7b40b681b1e6074a6a66dd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6b29b6de426a4ca89230d201c5924321, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a32e2fa9197d4b75a682d8aa4f70aa60, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c] to archive 2024-11-21T00:16:37,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:16:37,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6708abb5c53d44afaa042bbedc4e57b6 2024-11-21T00:16:37,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6662bb838f6043f1818628d8174f6ceb 2024-11-21T00:16:37,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a93f6df64d8144319d421ef50b7f9737 2024-11-21T00:16:37,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/019e0223f95f4a27b9c8b16c0c50fdbd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/019e0223f95f4a27b9c8b16c0c50fdbd 2024-11-21T00:16:37,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/bf9e846a52e6455ab01b8599a034f4e5 2024-11-21T00:16:37,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/ce662d7153a54aa98ccccc99f1f1625c 2024-11-21T00:16:37,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2e21c4d53f2947ad9bd11bd0f6774a0b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2e21c4d53f2947ad9bd11bd0f6774a0b 2024-11-21T00:16:37,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/be42db6aab3748afac5f4774ec0a0c6b 2024-11-21T00:16:37,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b1b6cd7df174d7e941f0c2e839cfc36 2024-11-21T00:16:37,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/645303fbe9a14589ba758db1e404ea07 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/645303fbe9a14589ba758db1e404ea07 2024-11-21T00:16:37,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/0ca6162230f64bb289473525331a1de3 2024-11-21T00:16:37,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/867a64bf4935455fbcab598f7b16d07f 2024-11-21T00:16:37,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/dfad404c2b5249ae9a150630377416aa 2024-11-21T00:16:37,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a6a292389cb14e2f9d0e9aee70b41295 2024-11-21T00:16:37,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/94585dc107dc48999095da1991306bcd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/94585dc107dc48999095da1991306bcd 2024-11-21T00:16:37,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/5aea29bd247b4bcaa3d6d7d38f1ba136 2024-11-21T00:16:37,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/15b098ef54f147fa822fef89dddaceb1 2024-11-21T00:16:37,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/417a19f2ff7b40b681b1e6074a6a66dd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/417a19f2ff7b40b681b1e6074a6a66dd 2024-11-21T00:16:37,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/d31c5e64c16e4f4885487658b0b66d35 2024-11-21T00:16:37,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/f527b64846bc4925988b45ddb04614c4 2024-11-21T00:16:37,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6b29b6de426a4ca89230d201c5924321 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/6b29b6de426a4ca89230d201c5924321 2024-11-21T00:16:37,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/7cbfcb5ebc9f42b3b5f734302fee23c2 2024-11-21T00:16:37,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/1e146ca950364917b1831eba25c0a8a9 2024-11-21T00:16:37,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a32e2fa9197d4b75a682d8aa4f70aa60 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/a32e2fa9197d4b75a682d8aa4f70aa60 2024-11-21T00:16:37,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/2b85b931565647539af4bf0e7b363dde 2024-11-21T00:16:37,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/4dbd9b462fa645adb8986dc37d53335c 2024-11-21T00:16:37,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/32915d9b82ee43ac8132694c73690b1c 2024-11-21T00:16:37,321 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-11-21T00:16:37,321 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. 2024-11-21T00:16:37,321 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for ec77d7e5be4bbf0794947ca9843b822c: Waiting for close lock at 1732148195983Running coprocessor pre-close hooks at 1732148195983Disabling compacts and flushes for region at 1732148195983Disabling writes for close at 1732148195983Obtaining lock to block concurrent updates at 1732148195983Preparing flush snapshotting stores in ec77d7e5be4bbf0794947ca9843b822c at 1732148195983Finished memstore snapshotting TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732148195984 (+1 ms)Flushing stores of TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c. at 1732148195984Flushing ec77d7e5be4bbf0794947ca9843b822c/A: creating writer at 1732148195984Flushing ec77d7e5be4bbf0794947ca9843b822c/A: appending metadata at 1732148195987 (+3 ms)Flushing ec77d7e5be4bbf0794947ca9843b822c/A: closing flushed file at 1732148195987Flushing ec77d7e5be4bbf0794947ca9843b822c/B: creating writer at 1732148196400 (+413 ms)Flushing ec77d7e5be4bbf0794947ca9843b822c/B: appending metadata at 1732148196403 (+3 ms)Flushing ec77d7e5be4bbf0794947ca9843b822c/B: closing flushed file at 1732148196403Flushing ec77d7e5be4bbf0794947ca9843b822c/C: creating writer at 1732148196811 (+408 ms)Flushing ec77d7e5be4bbf0794947ca9843b822c/C: appending metadata at 1732148196813 (+2 ms)Flushing ec77d7e5be4bbf0794947ca9843b822c/C: closing flushed file at 1732148196813Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@266dd1ac: reopening flushed file at 1732148197222 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17882872: reopening flushed file at 1732148197227 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@209b5d48: reopening flushed file at 1732148197231 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ec77d7e5be4bbf0794947ca9843b822c in 1253ms, sequenceid=469, compaction requested=true at 1732148197237 (+6 ms)Writing region close event to WAL at 1732148197318 (+81 ms)Running coprocessor post-close hooks at 1732148197321 (+3 ms)Closed at 1732148197321 2024-11-21T00:16:37,323 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:37,323 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ec77d7e5be4bbf0794947ca9843b822c, regionState=CLOSED 2024-11-21T00:16:37,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:37,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T00:16:37,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure ec77d7e5be4bbf0794947ca9843b822c, server=5ed4808ef0e6,40705,1732148113864 in 1.4940 sec 2024-11-21T00:16:37,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-21T00:16:37,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec77d7e5be4bbf0794947ca9843b822c, UNASSIGN in 1.4990 sec 2024-11-21T00:16:37,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T00:16:37,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5040 sec 2024-11-21T00:16:37,332 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148197332"}]},"ts":"1732148197332"} 2024-11-21T00:16:37,334 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:16:37,334 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:16:37,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-11-21T00:16:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T00:16:37,966 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:37,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:16:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:37,968 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T00:16:37,969 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:37,973 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:37,975 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/recovered.edits] 2024-11-21T00:16:37,977 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5b4e7219f60e4f2a9f60a311e74b26b1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/5b4e7219f60e4f2a9f60a311e74b26b1 2024-11-21T00:16:37,978 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/768c70334bd14f3b99b04a24e32d6b12 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/768c70334bd14f3b99b04a24e32d6b12 2024-11-21T00:16:37,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/d403a45411bf42b4aac4e4ba07998d96 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/d403a45411bf42b4aac4e4ba07998d96 2024-11-21T00:16:37,981 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e832ed4683244e80af84e361cd616577 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/A/e832ed4683244e80af84e361cd616577 2024-11-21T00:16:37,987 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1722dce54cfc4b6f915c658dcedf59f2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/1722dce54cfc4b6f915c658dcedf59f2 2024-11-21T00:16:37,988 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/8045936671234c6e9d5a7765a29cfda9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/8045936671234c6e9d5a7765a29cfda9 2024-11-21T00:16:37,989 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9d05bf68f1b7424abb99dd6e4d297f00 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/9d05bf68f1b7424abb99dd6e4d297f00 2024-11-21T00:16:37,990 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d48cbe6df2b047efaaeca255cd680f66 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/B/d48cbe6df2b047efaaeca255cd680f66 2024-11-21T00:16:37,992 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/3fc7ea1872a648ed94ce4bb151ddb2a0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/3fc7ea1872a648ed94ce4bb151ddb2a0 2024-11-21T00:16:37,993 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/58ba6a3e3d8d4920ae97f3910d704ee5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/58ba6a3e3d8d4920ae97f3910d704ee5 2024-11-21T00:16:37,994 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/c79b800b06e2423a9929d3b714eabf0c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/c79b800b06e2423a9929d3b714eabf0c 2024-11-21T00:16:37,995 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/db0a1d38e1c447228285343a449bf2aa to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/C/db0a1d38e1c447228285343a449bf2aa 2024-11-21T00:16:37,997 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/recovered.edits/472.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c/recovered.edits/472.seqid 2024-11-21T00:16:37,998 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/ec77d7e5be4bbf0794947ca9843b822c 2024-11-21T00:16:37,998 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:16:38,000 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:38,004 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:16:38,006 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:16:38,011 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:38,011 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:16:38,012 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148198011"}]},"ts":"9223372036854775807"} 2024-11-21T00:16:38,015 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:16:38,015 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ec77d7e5be4bbf0794947ca9843b822c, NAME => 'TestAcidGuarantees,,1732148170841.ec77d7e5be4bbf0794947ca9843b822c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:16:38,015 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:16:38,015 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148198015"}]},"ts":"9223372036854775807"} 2024-11-21T00:16:38,017 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:16:38,019 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:38,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-21T00:16:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T00:16:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:16:38,079 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:38,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 254), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=930 (was 877) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1959 (was 761) - AvailableMemoryMB LEAK? - 2024-11-21T00:16:38,100 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=930, ProcessCount=11, AvailableMemoryMB=1957 2024-11-21T00:16:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:16:38,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:16:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:38,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:16:38,105 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:38,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-21T00:16:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:38,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:16:38,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742154_1330 (size=1061) 2024-11-21T00:16:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:38,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:16:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742155_1331 (size=53) 2024-11-21T00:16:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 719d85aff3bf5e2cc4338f16a05e06b6, disabling compactions & flushes 2024-11-21T00:16:38,922 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. after waiting 0 ms 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:38,922 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:38,922 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 719d85aff3bf5e2cc4338f16a05e06b6: Waiting for close lock at 1732148198922Disabling compacts and flushes for region at 1732148198922Disabling writes for close at 1732148198922Writing region close event to WAL at 1732148198922Closed at 1732148198922 2024-11-21T00:16:38,923 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:16:38,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148198923"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148198923"}]},"ts":"1732148198923"} 2024-11-21T00:16:38,925 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:16:38,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:16:38,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148198926"}]},"ts":"1732148198926"} 2024-11-21T00:16:38,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:16:38,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, ASSIGN}] 2024-11-21T00:16:38,929 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, ASSIGN 2024-11-21T00:16:38,929 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:16:39,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:39,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, ASSIGN because future has completed 2024-11-21T00:16:39,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:39,235 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:39,236 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:16:39,236 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,236 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:16:39,236 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,236 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,238 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,239 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:39,239 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName A 2024-11-21T00:16:39,239 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:39,240 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:39,240 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,241 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:39,241 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName B 2024-11-21T00:16:39,241 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:39,241 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:39,241 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,242 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:39,243 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName C 2024-11-21T00:16:39,243 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:39,243 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:39,243 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:39,243 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,244 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,244 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,245 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,245 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,246 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:16:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:39,247 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,249 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:16:39,249 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 719d85aff3bf5e2cc4338f16a05e06b6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65677131, jitterRate=-0.021334484219551086}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:16:39,249 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:39,250 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 719d85aff3bf5e2cc4338f16a05e06b6: Running coprocessor pre-open hook at 1732148199236Writing region info on filesystem at 1732148199236Initializing all the Stores at 1732148199237 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148199237Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148199237Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148199237Cleaning up temporary data from old regions at 1732148199245 (+8 ms)Running coprocessor post-open hooks at 1732148199250 (+5 ms)Region opened successfully at 1732148199250 2024-11-21T00:16:39,251 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., pid=91, masterSystemTime=1732148199233 2024-11-21T00:16:39,253 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:39,253 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:39,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:39,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:39,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T00:16:39,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 in 174 msec 2024-11-21T00:16:39,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-21T00:16:39,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, ASSIGN in 328 msec 2024-11-21T00:16:39,259 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:16:39,259 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148199259"}]},"ts":"1732148199259"} 2024-11-21T00:16:39,260 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:16:39,261 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:16:39,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-21T00:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T00:16:40,257 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:40,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:40,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:40,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:40,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685dc8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:40,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa1bebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:40,266 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:40,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:40,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:16:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:16:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7aeae733 2024-11-21T00:16:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:16:40,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:16:40,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:16:40,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:16:40,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742156_1332 (size=1097) 2024-11-21T00:16:40,401 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T00:16:40,401 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T00:16:40,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:16:40,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE}] 2024-11-21T00:16:40,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE 2024-11-21T00:16:40,414 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:40,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE because future has completed 2024-11-21T00:16:40,416 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:16:40,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:40,570 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,570 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:16:40,570 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 719d85aff3bf5e2cc4338f16a05e06b6, disabling compactions & flushes 2024-11-21T00:16:40,571 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,571 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,571 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. after waiting 0 ms 2024-11-21T00:16:40,571 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,600 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:16:40,604 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,605 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 719d85aff3bf5e2cc4338f16a05e06b6: Waiting for close lock at 1732148200570Running coprocessor pre-close hooks at 1732148200570Disabling compacts and flushes for region at 1732148200570Disabling writes for close at 1732148200571 (+1 ms)Writing region close event to WAL at 1732148200588 (+17 ms)Running coprocessor post-close hooks at 1732148200604 (+16 ms)Closed at 1732148200604 2024-11-21T00:16:40,605 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 719d85aff3bf5e2cc4338f16a05e06b6 to self. 2024-11-21T00:16:40,618 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,619 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=CLOSED 2024-11-21T00:16:40,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:40,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T00:16:40,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 in 209 msec 2024-11-21T00:16:40,639 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=true 2024-11-21T00:16:40,789 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:40,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE because future has completed 2024-11-21T00:16:40,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:16:40,962 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,962 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:16:40,962 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,963 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:16:40,963 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,963 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,965 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,966 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:40,967 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName A 2024-11-21T00:16:40,969 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:40,969 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:40,970 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,972 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:40,972 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName B 2024-11-21T00:16:40,972 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:40,973 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:40,973 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,975 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:16:40,975 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 719d85aff3bf5e2cc4338f16a05e06b6 columnFamilyName C 2024-11-21T00:16:40,975 DEBUG [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:40,976 INFO [StoreOpener-719d85aff3bf5e2cc4338f16a05e06b6-1 {}] regionserver.HStore(327): Store=719d85aff3bf5e2cc4338f16a05e06b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:16:40,976 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,977 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,977 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,979 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,980 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,980 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,981 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:16:40,982 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,983 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 719d85aff3bf5e2cc4338f16a05e06b6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67786098, jitterRate=0.010091572999954224}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:16:40,983 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:40,984 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 719d85aff3bf5e2cc4338f16a05e06b6: Running coprocessor pre-open hook at 1732148200963Writing region info on filesystem at 1732148200963Initializing all the Stores at 1732148200964 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148200964Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148200964Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148200965 (+1 ms)Cleaning up temporary data from old regions at 1732148200980 (+15 ms)Running coprocessor post-open hooks at 1732148200983 (+3 ms)Region opened successfully at 1732148200984 (+1 ms) 2024-11-21T00:16:40,985 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., pid=96, masterSystemTime=1732148200957 2024-11-21T00:16:40,987 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,987 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:40,992 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:40,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:16:40,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-21T00:16:40,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 in 201 msec 2024-11-21T00:16:41,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T00:16:41,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, REOPEN/MOVE in 587 msec 2024-11-21T00:16:41,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-21T00:16:41,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 599 msec 2024-11-21T00:16:41,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 656 msec 2024-11-21T00:16:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T00:16:41,009 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:41,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,011 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7b7c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530212c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,020 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426b351d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,026 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78851269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,028 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1172b768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f6795c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f440c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,035 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473e5349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217a976c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,040 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf6fbd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,043 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d1e667, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,048 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac09d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,051 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53961333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,055 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f13dcef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,057 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb24829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,061 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae811c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e075b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:16:41,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:16:41,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa8fa88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:16:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:16:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:16:41,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:16:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e49326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:16:41,071 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-21T00:16:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:41,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:41,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:41,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:41,080 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,080 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,102 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,103 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,104 DEBUG [Thread-1511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,121 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,122 DEBUG [Thread-1519 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,122 DEBUG [Thread-1519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:41,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,129 DEBUG [Thread-1522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,131 DEBUG [Thread-1522 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,131 DEBUG [Thread-1522 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,140 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,142 DEBUG [Thread-1513 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,142 DEBUG [Thread-1513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,144 DEBUG [Thread-1517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,145 DEBUG [Thread-1517 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,146 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:16:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:16:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213e0fcec86116471fa8975a130e5286a0_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148201121/Put/seqid=0 2024-11-21T00:16:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,150 DEBUG [Thread-1517 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:16:41,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:16:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:16:41,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742157_1333 (size=12154) 2024-11-21T00:16:41,161 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213e0fcec86116471fa8975a130e5286a0_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213e0fcec86116471fa8975a130e5286a0_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:41,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7cfcf3ab5b1c44abab1d75dfca1b8731, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:41,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7cfcf3ab5b1c44abab1d75dfca1b8731 is 175, key is test_row_0/A:col10/1732148201121/Put/seqid=0 2024-11-21T00:16:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:41,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148201146, max timestamp=1732148201183Num uniques 859; 2024-11-21T00:16:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148261167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:41,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148201146, max timestamp=1732148201183Num uniques 859; 2024-11-21T00:16:41,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732148201146, max timestamp=1732148201183Num uniques 859; 2024-11-21T00:16:41,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:41,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742158_1334 (size=30955) 2024-11-21T00:16:41,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7cfcf3ab5b1c44abab1d75dfca1b8731 2024-11-21T00:16:41,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:41,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b1bb5369016148a5af99ce7303265900 is 50, key is test_row_0/B:col10/1732148201121/Put/seqid=0 2024-11-21T00:16:41,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742159_1335 (size=12001) 2024-11-21T00:16:41,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b1bb5369016148a5af99ce7303265900 2024-11-21T00:16:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148261241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148261242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148261257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148261259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/0ca14bea31c143bf9550c84b6417b09f is 50, key is test_row_0/C:col10/1732148201121/Put/seqid=0 2024-11-21T00:16:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148261297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742160_1336 (size=12001) 2024-11-21T00:16:41,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:41,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:41,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148261372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148261372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148261376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148261377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148261517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148261587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148261587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148261596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148261596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:41,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/0ca14bea31c143bf9550c84b6417b09f 2024-11-21T00:16:41,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7cfcf3ab5b1c44abab1d75dfca1b8731 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731 2024-11-21T00:16:41,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731, entries=150, sequenceid=14, filesize=30.2 K 2024-11-21T00:16:41,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b1bb5369016148a5af99ce7303265900 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900 2024-11-21T00:16:41,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T00:16:41,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/0ca14bea31c143bf9550c84b6417b09f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f 2024-11-21T00:16:41,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T00:16:41,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 719d85aff3bf5e2cc4338f16a05e06b6 in 644ms, sequenceid=14, compaction requested=false 2024-11-21T00:16:41,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:41,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:41,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:41,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:41,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:41,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:41,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:41,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121330062e5bf5140289e83f3337fe3616c_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148201237/Put/seqid=0 2024-11-21T00:16:41,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742161_1337 (size=14594) 2024-11-21T00:16:41,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:41,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148261907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148261911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148261916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148261916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148261920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121330062e5bf5140289e83f3337fe3616c_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121330062e5bf5140289e83f3337fe3616c_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:41,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d37ac67ffbd249a082b213f610db1328, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:41,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d37ac67ffbd249a082b213f610db1328 is 175, key is test_row_0/A:col10/1732148201237/Put/seqid=0 2024-11-21T00:16:41,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742162_1338 (size=39549) 2024-11-21T00:16:41,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d37ac67ffbd249a082b213f610db1328 2024-11-21T00:16:41,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/234b9f84175845fdbe2dfc72bc6139eb is 50, key is test_row_0/B:col10/1732148201237/Put/seqid=0 2024-11-21T00:16:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742163_1339 (size=12001) 2024-11-21T00:16:41,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/234b9f84175845fdbe2dfc72bc6139eb 2024-11-21T00:16:42,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:42,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:42,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fc86caddfd804a59963fe395f09b4b41 is 50, key is test_row_0/C:col10/1732148201237/Put/seqid=0 2024-11-21T00:16:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148262041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742164_1340 (size=12001) 2024-11-21T00:16:42,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fc86caddfd804a59963fe395f09b4b41 2024-11-21T00:16:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d37ac67ffbd249a082b213f610db1328 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328 2024-11-21T00:16:42,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328, entries=200, sequenceid=50, filesize=38.6 K 2024-11-21T00:16:42,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/234b9f84175845fdbe2dfc72bc6139eb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb 2024-11-21T00:16:42,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:16:42,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T00:16:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fc86caddfd804a59963fe395f09b4b41 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41 2024-11-21T00:16:42,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T00:16:42,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 719d85aff3bf5e2cc4338f16a05e06b6 in 281ms, sequenceid=50, compaction requested=false 2024-11-21T00:16:42,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:42,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:42,170 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:42,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112146ae7c47d71443e39fd1221408e970b8_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148201918/Put/seqid=0 2024-11-21T00:16:42,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742165_1341 (size=12154) 2024-11-21T00:16:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:42,195 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112146ae7c47d71443e39fd1221408e970b8_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112146ae7c47d71443e39fd1221408e970b8_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/91c477e9345b411189ed4ab5ca8ba499, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/91c477e9345b411189ed4ab5ca8ba499 is 175, key is test_row_0/A:col10/1732148201918/Put/seqid=0 2024-11-21T00:16:42,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742166_1342 (size=30955) 2024-11-21T00:16:42,202 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/91c477e9345b411189ed4ab5ca8ba499 2024-11-21T00:16:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d7e0e3402ae942868a672e2f1087b1af is 50, key is test_row_0/B:col10/1732148201918/Put/seqid=0 2024-11-21T00:16:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742167_1343 (size=12001) 2024-11-21T00:16:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:42,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:42,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148202387, max timestamp=1732148202446Num uniques 859; 2024-11-21T00:16:42,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:42,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148202387, max timestamp=1732148202446Num uniques 859; 2024-11-21T00:16:42,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148202387, max timestamp=1732148202446Num uniques 859; 2024-11-21T00:16:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148262511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148262521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148262523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148262524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148262530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,615 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d7e0e3402ae942868a672e2f1087b1af 2024-11-21T00:16:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/c860e348c4cc4f01aea58636b37edf69 is 50, key is test_row_0/C:col10/1732148201918/Put/seqid=0 2024-11-21T00:16:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148262636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148262636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148262647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148262647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148262647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:42,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742168_1344 (size=12001) 2024-11-21T00:16:42,674 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/c860e348c4cc4f01aea58636b37edf69 2024-11-21T00:16:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/91c477e9345b411189ed4ab5ca8ba499 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499 2024-11-21T00:16:42,687 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499, entries=150, sequenceid=59, filesize=30.2 K 2024-11-21T00:16:42,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d7e0e3402ae942868a672e2f1087b1af as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af 2024-11-21T00:16:42,697 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T00:16:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/c860e348c4cc4f01aea58636b37edf69 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69 2024-11-21T00:16:42,716 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T00:16:42,725 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 719d85aff3bf5e2cc4338f16a05e06b6 in 555ms, sequenceid=59, compaction requested=true 2024-11-21T00:16:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:42,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-21T00:16:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-21T00:16:42,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T00:16:42,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-21T00:16:42,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6590 sec 2024-11-21T00:16:42,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148202519, max timestamp=1732148202860Num uniques 1068; 2024-11-21T00:16:42,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-21T00:16:42,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148202519, max timestamp=1732148202860Num uniques 1068; 2024-11-21T00:16:42,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-21T00:16:42,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-11-21T00:16:42,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:42,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:42,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:42,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:42,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:42,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:16:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:42,876 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:16:42,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112159e051070c56440ca70fba44f6801f63_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742169_1345 (size=12154) 2024-11-21T00:16:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148262956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148262972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732148202943, max timestamp=1732148202997Num uniques 859; 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732148202943, max timestamp=1732148202997Num uniques 859; 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732148202943, max timestamp=1732148202997Num uniques 859; 2024-11-21T00:16:43,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T00:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148263089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148263092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148263094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148263107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148263107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:16:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148263210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148263211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,218 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T00:16:43,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:43,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:43,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148263217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148263326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148263327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,369 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:43,375 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112159e051070c56440ca70fba44f6801f63_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159e051070c56440ca70fba44f6801f63_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:43,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/18518efc49304025b3f10f06b5bfa1d1, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:43,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/18518efc49304025b3f10f06b5bfa1d1 is 175, key is test_row_0/A:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:43,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742170_1346 (size=30955) 2024-11-21T00:16:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148263428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148263428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148263447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:43,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148263646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148263647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148263747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148263747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148263768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:43,821 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/18518efc49304025b3f10f06b5bfa1d1 2024-11-21T00:16:43,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/89f9fa9b8cc64e908be45df6aba4c7cb is 50, key is test_row_0/B:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742171_1347 (size=12001) 2024-11-21T00:16:43,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/89f9fa9b8cc64e908be45df6aba4c7cb 2024-11-21T00:16:43,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f19e1efe72a940fe92629662a454077d is 50, key is test_row_0/C:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:43,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742172_1348 (size=12001) 2024-11-21T00:16:43,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:43,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:43,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148264166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148264166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f19e1efe72a940fe92629662a454077d 2024-11-21T00:16:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/18518efc49304025b3f10f06b5bfa1d1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1 2024-11-21T00:16:44,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1, entries=150, sequenceid=97, filesize=30.2 K 2024-11-21T00:16:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/89f9fa9b8cc64e908be45df6aba4c7cb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb 2024-11-21T00:16:44,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148264267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148264267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T00:16:44,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f19e1efe72a940fe92629662a454077d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d 2024-11-21T00:16:44,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T00:16:44,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.83 KB/188240, currentSize=120.76 KB/123660 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1437ms, sequenceid=97, compaction requested=true 2024-11-21T00:16:44,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:44,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:44,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:44,303 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,307 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:44,309 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:44,309 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:16:44,309 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,309 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=129.3 K 2024-11-21T00:16:44,310 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,310 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1] 2024-11-21T00:16:44,310 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:44,310 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:16:44,310 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,311 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=46.9 K 2024-11-21T00:16:44,311 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cfcf3ab5b1c44abab1d75dfca1b8731, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148201118 2024-11-21T00:16:44,311 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b1bb5369016148a5af99ce7303265900, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148201118 2024-11-21T00:16:44,311 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37ac67ffbd249a082b213f610db1328, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148201237 2024-11-21T00:16:44,312 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c477e9345b411189ed4ab5ca8ba499, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148201881 2024-11-21T00:16:44,312 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 234b9f84175845fdbe2dfc72bc6139eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148201237 2024-11-21T00:16:44,312 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18518efc49304025b3f10f06b5bfa1d1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:44,312 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e0e3402ae942868a672e2f1087b1af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148201881 2024-11-21T00:16:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:44,313 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f9fa9b8cc64e908be45df6aba4c7cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:44,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:44,339 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:44,340 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b38cc86f4c5e4c5f83d84d50d155f23b is 50, key is test_row_0/B:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:44,346 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:44,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121937a4d72be7b448992e52571b17cfbbe_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148203045/Put/seqid=0 2024-11-21T00:16:44,371 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f3b3db49f87345f098677d8ef9bc5935_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:44,373 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f3b3db49f87345f098677d8ef9bc5935_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:44,374 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f3b3db49f87345f098677d8ef9bc5935_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742173_1349 (size=12139) 2024-11-21T00:16:44,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742174_1350 (size=14694) 2024-11-21T00:16:44,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742175_1351 (size=4469) 2024-11-21T00:16:44,420 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#295 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:44,420 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/edc69770e4a24d908207a6d9bd079741 is 175, key is test_row_0/A:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:44,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742176_1352 (size=31093) 2024-11-21T00:16:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148264434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148264557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148264776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:44,805 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b38cc86f4c5e4c5f83d84d50d155f23b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b38cc86f4c5e4c5f83d84d50d155f23b 2024-11-21T00:16:44,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:44,811 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into b38cc86f4c5e4c5f83d84d50d155f23b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:44,811 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:44,811 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=12, startTime=1732148204303; duration=0sec 2024-11-21T00:16:44,811 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:44,811 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:16:44,811 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:44,821 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:44,821 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:16:44,821 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,821 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=46.9 K 2024-11-21T00:16:44,822 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca14bea31c143bf9550c84b6417b09f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732148201118 2024-11-21T00:16:44,823 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fc86caddfd804a59963fe395f09b4b41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732148201237 2024-11-21T00:16:44,823 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c860e348c4cc4f01aea58636b37edf69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148201881 2024-11-21T00:16:44,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121937a4d72be7b448992e52571b17cfbbe_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121937a4d72be7b448992e52571b17cfbbe_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:44,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/a474a1546c184796abec63e210bfad21, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:44,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/a474a1546c184796abec63e210bfad21 is 175, key is test_row_0/A:col10/1732148203045/Put/seqid=0 2024-11-21T00:16:44,828 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f19e1efe72a940fe92629662a454077d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:44,845 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/edc69770e4a24d908207a6d9bd079741 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741 2024-11-21T00:16:44,851 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into edc69770e4a24d908207a6d9bd079741(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:44,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:44,851 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=12, startTime=1732148204303; duration=0sec 2024-11-21T00:16:44,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:44,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:16:44,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742177_1353 (size=39649) 2024-11-21T00:16:44,863 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:44,863 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/09549b18df5e4baabed99b627200dd0c is 50, key is test_row_0/C:col10/1732148202519/Put/seqid=0 2024-11-21T00:16:44,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742178_1354 (size=12139) 2024-11-21T00:16:44,915 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/09549b18df5e4baabed99b627200dd0c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/09549b18df5e4baabed99b627200dd0c 2024-11-21T00:16:44,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:44,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:44,956 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into 09549b18df5e4baabed99b627200dd0c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:44,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:44,956 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=12, startTime=1732148204307; duration=0sec 2024-11-21T00:16:44,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:44,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:16:45,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148265107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148265186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,206 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148265186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,208 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/a474a1546c184796abec63e210bfad21 2024-11-21T00:16:45,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/23b5a8fd66684ba5bff454bf5270007d is 50, key is test_row_0/B:col10/1732148203045/Put/seqid=0 2024-11-21T00:16:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148265297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148265297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,303 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,306 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742179_1355 (size=12101) 2024-11-21T00:16:45,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/23b5a8fd66684ba5bff454bf5270007d 2024-11-21T00:16:45,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/6e3655c11657401dbece11f58fe6fa72 is 50, key is test_row_0/C:col10/1732148203045/Put/seqid=0 2024-11-21T00:16:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:45,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742180_1356 (size=12101) 2024-11-21T00:16:45,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148265636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:45,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:45,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/6e3655c11657401dbece11f58fe6fa72 2024-11-21T00:16:45,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/a474a1546c184796abec63e210bfad21 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21 2024-11-21T00:16:45,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21, entries=200, sequenceid=131, filesize=38.7 K 2024-11-21T00:16:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/23b5a8fd66684ba5bff454bf5270007d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d 2024-11-21T00:16:45,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T00:16:45,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/6e3655c11657401dbece11f58fe6fa72 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72 2024-11-21T00:16:45,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T00:16:45,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1507ms, sequenceid=131, compaction requested=false 2024-11-21T00:16:45,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:45,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:16:45,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:45,872 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cc8ba20974214b2aa2b7c1d863426239_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148204429/Put/seqid=0 2024-11-21T00:16:45,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742181_1357 (size=12304) 2024-11-21T00:16:46,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:46,342 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cc8ba20974214b2aa2b7c1d863426239_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc8ba20974214b2aa2b7c1d863426239_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7c982b83155a49c893cb1050d1871c28, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:46,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7c982b83155a49c893cb1050d1871c28 is 175, key is test_row_0/A:col10/1732148204429/Put/seqid=0 2024-11-21T00:16:46,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742182_1358 (size=31105) 2024-11-21T00:16:46,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:46,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7c982b83155a49c893cb1050d1871c28 2024-11-21T00:16:46,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/cc14b587fb6146e3916a64a1d995d506 is 50, key is test_row_0/B:col10/1732148204429/Put/seqid=0 2024-11-21T00:16:46,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742183_1359 (size=12151) 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148206742, max timestamp=1732148206798Num uniques 859; 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148206742, max timestamp=1732148206798Num uniques 859; 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148206742, max timestamp=1732148206798Num uniques 859; 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:46,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:16:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148266972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148267087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,174 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/cc14b587fb6146e3916a64a1d995d506 2024-11-21T00:16:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/b81b35940b9649d58d3a47fddd781f44 is 50, key is test_row_0/C:col10/1732148204429/Put/seqid=0 2024-11-21T00:16:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742184_1360 (size=12151) 2024-11-21T00:16:47,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148267217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148267217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,224 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,225 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148267297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148267326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148267326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,334 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,335 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:47,587 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/b81b35940b9649d58d3a47fddd781f44 2024-11-21T00:16:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7c982b83155a49c893cb1050d1871c28 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28 2024-11-21T00:16:47,598 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28, entries=150, sequenceid=145, filesize=30.4 K 2024-11-21T00:16:47,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/cc14b587fb6146e3916a64a1d995d506 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506 2024-11-21T00:16:47,603 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T00:16:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/b81b35940b9649d58d3a47fddd781f44 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44 2024-11-21T00:16:47,608 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T00:16:47,608 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1736ms, sequenceid=145, compaction requested=true 2024-11-21T00:16:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T00:16:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T00:16:47,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T00:16:47,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3860 sec 2024-11-21T00:16:47,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 4.3900 sec 2024-11-21T00:16:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:47,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:47,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:47,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215bfa6ffc3aa54c4ab197bf190ae57755_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742185_1361 (size=14794) 2024-11-21T00:16:47,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:47,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215bfa6ffc3aa54c4ab197bf190ae57755_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215bfa6ffc3aa54c4ab197bf190ae57755_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:47,641 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d0b362a0ebba469a93a4e8cf21163508, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:47,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d0b362a0ebba469a93a4e8cf21163508 is 175, key is test_row_0/A:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:47,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742186_1362 (size=39749) 2024-11-21T00:16:47,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d0b362a0ebba469a93a4e8cf21163508 2024-11-21T00:16:47,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/62e631ecbace4a7bb30d323f16a40fcc is 50, key is test_row_0/B:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742187_1363 (size=12151) 2024-11-21T00:16:47,785 INFO [AsyncFSWAL-0-hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b-prefix:5ed4808ef0e6,40705,1732148113864 {}] wal.AbstractFSWAL(1368): Slow sync cost: 103 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45765,DS-9aed6128-6019-4e27-b8d1-7d3c3156bc3a,DISK]] 2024-11-21T00:16:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148267786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148267907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:48,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/62e631ecbace4a7bb30d323f16a40fcc 2024-11-21T00:16:48,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4ce86762d22847cf9cbac253d6f5d1c4 is 50, key is test_row_0/C:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:48,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742188_1364 (size=12151) 2024-11-21T00:16:48,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148268130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148268446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:48,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4ce86762d22847cf9cbac253d6f5d1c4 2024-11-21T00:16:48,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/d0b362a0ebba469a93a4e8cf21163508 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508 2024-11-21T00:16:48,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508, entries=200, sequenceid=181, filesize=38.8 K 2024-11-21T00:16:48,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/62e631ecbace4a7bb30d323f16a40fcc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc 2024-11-21T00:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc, entries=150, sequenceid=181, filesize=11.9 K 2024-11-21T00:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4ce86762d22847cf9cbac253d6f5d1c4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4 2024-11-21T00:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4, entries=150, sequenceid=181, filesize=11.9 K 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 869ms, sequenceid=181, compaction requested=true 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:48,495 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:48,495 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:48,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,496 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141596 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:48,496 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,497 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:48,497 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:48,497 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b38cc86f4c5e4c5f83d84d50d155f23b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=47.4 K 2024-11-21T00:16:48,497 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=138.3 K 2024-11-21T00:16:48,497 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508] 2024-11-21T00:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b38cc86f4c5e4c5f83d84d50d155f23b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting edc69770e4a24d908207a6d9bd079741, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b5a8fd66684ba5bff454bf5270007d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148203045 2024-11-21T00:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a474a1546c184796abec63e210bfad21, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148203045 2024-11-21T00:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,497 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cc14b587fb6146e3916a64a1d995d506, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732148204345 2024-11-21T00:16:48,498 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c982b83155a49c893cb1050d1871c28, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732148204345 2024-11-21T00:16:48,498 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e631ecbace4a7bb30d323f16a40fcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:48,498 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b362a0ebba469a93a4e8cf21163508, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,503 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:48,504 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411219b3e2f21705b49609bf8e0c0d3d52ffa_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:48,504 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,505 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/209b4852f0ac4f2699f2f4f111a35163 is 50, key is test_row_0/B:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,507 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411219b3e2f21705b49609bf8e0c0d3d52ffa_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,507 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219b3e2f21705b49609bf8e0c0d3d52ffa_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742189_1365 (size=12425) 2024-11-21T00:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742190_1366 (size=4469) 2024-11-21T00:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,933 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,933 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/e7ff00f747064a748bef1703ba45a43b is 175, key is test_row_0/A:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,940 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/209b4852f0ac4f2699f2f4f111a35163 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/209b4852f0ac4f2699f2f4f111a35163 2024-11-21T00:16:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,948 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into 209b4852f0ac4f2699f2f4f111a35163(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,948 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:48,949 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=12, startTime=1732148208495; duration=0sec 2024-11-21T00:16:48,949 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:48,949 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:16:48,949 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,953 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:48,953 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:16:48,953 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,953 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/09549b18df5e4baabed99b627200dd0c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=47.4 K 2024-11-21T00:16:48,954 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 09549b18df5e4baabed99b627200dd0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732148202519 2024-11-21T00:16:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,955 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3655c11657401dbece11f58fe6fa72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732148203045 2024-11-21T00:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b81b35940b9649d58d3a47fddd781f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732148204345 2024-11-21T00:16:48,956 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce86762d22847cf9cbac253d6f5d1c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742191_1367 (size=31379) 2024-11-21T00:16:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,966 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/e7ff00f747064a748bef1703ba45a43b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b 2024-11-21T00:16:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,975 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#308 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,975 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into e7ff00f747064a748bef1703ba45a43b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:48,975 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:48,975 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=12, startTime=1732148208495; duration=0sec 2024-11-21T00:16:48,975 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:48,975 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/b0af1f0bf531434a9e7caa5edb5a17ad is 50, key is test_row_0/C:col10/1732148206860/Put/seqid=0 2024-11-21T00:16:48,976 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:16:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:16:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742192_1368 (size=12425) 2024-11-21T00:16:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212fae0eabc0fb4661a77960f8c15c34a9_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_1/A:col10/1732148208986/Put/seqid=0 2024-11-21T00:16:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,036 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/b0af1f0bf531434a9e7caa5edb5a17ad as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b0af1f0bf531434a9e7caa5edb5a17ad 2024-11-21T00:16:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742193_1369 (size=12304) 2024-11-21T00:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,061 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into b0af1f0bf531434a9e7caa5edb5a17ad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:49,061 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:49,061 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=12, startTime=1732148208495; duration=0sec 2024-11-21T00:16:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,061 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:49,061 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:16:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212fae0eabc0fb4661a77960f8c15c34a9_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fae0eabc0fb4661a77960f8c15c34a9_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,068 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7de067dcabbb49e7a51da09973badd6f, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7de067dcabbb49e7a51da09973badd6f is 175, key is test_row_1/A:col10/1732148208986/Put/seqid=0 2024-11-21T00:16:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742194_1370 (size=31101) 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:49,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148209199, max timestamp=1732148209262Num uniques 859; 2024-11-21T00:16:49,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:49,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148209199, max timestamp=1732148209262Num uniques 859; 2024-11-21T00:16:49,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:49,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148209199, max timestamp=1732148209262Num uniques 859; 2024-11-21T00:16:49,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148269441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:49,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7de067dcabbb49e7a51da09973badd6f 2024-11-21T00:16:49,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/a89b5de19694486c9e1b14d241382e7d is 50, key is test_row_1/B:col10/1732148208986/Put/seqid=0 2024-11-21T00:16:49,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742195_1371 (size=9757) 2024-11-21T00:16:49,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/a89b5de19694486c9e1b14d241382e7d 2024-11-21T00:16:49,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4fb353ada5444035be95dc84e7552ac7 is 50, key is test_row_1/C:col10/1732148208986/Put/seqid=0 2024-11-21T00:16:49,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742196_1372 (size=9757) 2024-11-21T00:16:49,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148269557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148269777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:49,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4fb353ada5444035be95dc84e7552ac7 2024-11-21T00:16:49,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/7de067dcabbb49e7a51da09973badd6f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f 2024-11-21T00:16:49,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f, entries=150, sequenceid=194, filesize=30.4 K 2024-11-21T00:16:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/a89b5de19694486c9e1b14d241382e7d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d 2024-11-21T00:16:49,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d, entries=100, sequenceid=194, filesize=9.5 K 2024-11-21T00:16:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/4fb353ada5444035be95dc84e7552ac7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7 2024-11-21T00:16:49,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7, entries=100, sequenceid=194, filesize=9.5 K 2024-11-21T00:16:49,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1002ms, sequenceid=194, compaction requested=false 2024-11-21T00:16:49,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:50,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:50,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:50,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b002554d090d4ff093c724c361b06495_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:50,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742197_1373 (size=14794) 2024-11-21T00:16:50,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148270163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148270277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148270486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:50,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:50,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b002554d090d4ff093c724c361b06495_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b002554d090d4ff093c724c361b06495_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:50,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/f3934c2f72ec4fc08de35f9ae827db8e, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:50,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/f3934c2f72ec4fc08de35f9ae827db8e is 175, key is test_row_0/A:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:50,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742198_1374 (size=39749) 2024-11-21T00:16:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148270806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:50,940 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/f3934c2f72ec4fc08de35f9ae827db8e 2024-11-21T00:16:50,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b3d1ca980f394827a010d014706c5c37 is 50, key is test_row_0/B:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742199_1375 (size=12151) 2024-11-21T00:16:51,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148271248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:51,249 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:51,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148271266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:51,276 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148271317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148271346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:51,348 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:51,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b3d1ca980f394827a010d014706c5c37 2024-11-21T00:16:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148271359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:51,365 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:16:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:51,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/a8198e05ef14448da3523b7a860db4f6 is 50, key is test_row_0/C:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:16:51,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:51,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742200_1376 (size=12151) 2024-11-21T00:16:51,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T00:16:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:16:51,403 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:51,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:51,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:16:51,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:16:51,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:16:51,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:16:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:51,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/a8198e05ef14448da3523b7a860db4f6 2024-11-21T00:16:51,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/f3934c2f72ec4fc08de35f9ae827db8e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e 2024-11-21T00:16:51,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e, entries=200, sequenceid=232, filesize=38.8 K 2024-11-21T00:16:51,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/b3d1ca980f394827a010d014706c5c37 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37 2024-11-21T00:16:51,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37, entries=150, sequenceid=232, filesize=11.9 K 2024-11-21T00:16:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/a8198e05ef14448da3523b7a860db4f6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6 2024-11-21T00:16:51,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-21T00:16:51,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1739ms, sequenceid=232, compaction requested=true 2024-11-21T00:16:51,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:51,843 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:51,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:51,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:51,843 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:51,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:51,848 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:16:51,848 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,849 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/209b4852f0ac4f2699f2f4f111a35163, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=33.5 K 2024-11-21T00:16:51,850 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 209b4852f0ac4f2699f2f4f111a35163, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:51,850 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a89b5de19694486c9e1b14d241382e7d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732148207681 2024-11-21T00:16:51,850 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:51,850 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:16:51,850 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,850 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=99.8 K 2024-11-21T00:16:51,850 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,850 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e] 2024-11-21T00:16:51,851 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d1ca980f394827a010d014706c5c37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:51,853 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ff00f747064a748bef1703ba45a43b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:51,854 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de067dcabbb49e7a51da09973badd6f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732148207669 2024-11-21T00:16:51,854 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3934c2f72ec4fc08de35f9ae827db8e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:51,869 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:51,870 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/2d316bdd6e474b15ab975207747567eb is 50, key is test_row_0/B:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:51,877 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:51,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:16:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,882 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:51,888 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c0b723dde92040978003ddd3dacfae1c_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:51,890 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c0b723dde92040978003ddd3dacfae1c_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:51,891 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c0b723dde92040978003ddd3dacfae1c_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218738cb19b15445aba3d177af59d415b3_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148210152/Put/seqid=0 2024-11-21T00:16:51,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742201_1377 (size=12527) 2024-11-21T00:16:51,910 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/2d316bdd6e474b15ab975207747567eb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/2d316bdd6e474b15ab975207747567eb 2024-11-21T00:16:51,917 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into 2d316bdd6e474b15ab975207747567eb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:51,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:51,917 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=13, startTime=1732148211843; duration=0sec 2024-11-21T00:16:51,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:51,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:16:51,917 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:51,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742202_1378 (size=4469) 2024-11-21T00:16:51,921 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:51,921 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:16:51,921 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:51,921 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b0af1f0bf531434a9e7caa5edb5a17ad, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=33.5 K 2024-11-21T00:16:51,922 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b0af1f0bf531434a9e7caa5edb5a17ad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732148206860 2024-11-21T00:16:51,922 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb353ada5444035be95dc84e7552ac7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732148207681 2024-11-21T00:16:51,923 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a8198e05ef14448da3523b7a860db4f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:51,927 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#316 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:51,928 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/1718fdbea748467f9d9f27ff4ba9882e is 175, key is test_row_0/A:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742203_1379 (size=12304) 2024-11-21T00:16:51,938 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:51,939 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/d7bc8e99829f40e996aa40e838f8dded is 50, key is test_row_0/C:col10/1732148209438/Put/seqid=0 2024-11-21T00:16:51,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742204_1380 (size=31481) 2024-11-21T00:16:51,970 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/1718fdbea748467f9d9f27ff4ba9882e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e 2024-11-21T00:16:51,977 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into 1718fdbea748467f9d9f27ff4ba9882e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:51,977 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:51,977 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=13, startTime=1732148211843; duration=0sec 2024-11-21T00:16:51,977 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:51,977 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:16:51,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742205_1381 (size=12527) 2024-11-21T00:16:52,008 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/d7bc8e99829f40e996aa40e838f8dded as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d7bc8e99829f40e996aa40e838f8dded 2024-11-21T00:16:52,020 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into d7bc8e99829f40e996aa40e838f8dded(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:52,020 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:52,020 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=13, startTime=1732148211847; duration=0sec 2024-11-21T00:16:52,020 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:52,020 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:16:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:16:52,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:52,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:52,350 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218738cb19b15445aba3d177af59d415b3_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218738cb19b15445aba3d177af59d415b3_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:52,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/ef0aa7fef1194092bdc7b922a193b6c2, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:52,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/ef0aa7fef1194092bdc7b922a193b6c2 is 175, key is test_row_0/A:col10/1732148210152/Put/seqid=0 2024-11-21T00:16:52,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742206_1382 (size=31105) 2024-11-21T00:16:52,380 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/ef0aa7fef1194092bdc7b922a193b6c2 2024-11-21T00:16:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9045304eaf7f487fa23dc8372235e0f9 is 50, key is test_row_0/B:col10/1732148210152/Put/seqid=0 2024-11-21T00:16:52,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742207_1383 (size=12151) 2024-11-21T00:16:52,424 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9045304eaf7f487fa23dc8372235e0f9 2024-11-21T00:16:52,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/ad83779d1b33478292fa3866b87d9c92 is 50, key is test_row_0/C:col10/1732148210152/Put/seqid=0 2024-11-21T00:16:52,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742208_1384 (size=12151) 2024-11-21T00:16:52,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212452, max timestamp=1732148212480Num uniques 859; 2024-11-21T00:16:52,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212452, max timestamp=1732148212480Num uniques 859; 2024-11-21T00:16:52,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,486 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/ad83779d1b33478292fa3866b87d9c92 2024-11-21T00:16:52,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212452, max timestamp=1732148212480Num uniques 859; 2024-11-21T00:16:52,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/ef0aa7fef1194092bdc7b922a193b6c2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2 2024-11-21T00:16:52,499 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2, entries=150, sequenceid=242, filesize=30.4 K 2024-11-21T00:16:52,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9045304eaf7f487fa23dc8372235e0f9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9 2024-11-21T00:16:52,514 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9, entries=150, sequenceid=242, filesize=11.9 K 2024-11-21T00:16:52,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/ad83779d1b33478292fa3866b87d9c92 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92 2024-11-21T00:16:52,525 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92, entries=150, sequenceid=242, filesize=11.9 K 2024-11-21T00:16:52,526 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 644ms, sequenceid=242, compaction requested=false 2024-11-21T00:16:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T00:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T00:16:52,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T00:16:52,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1200 sec 2024-11-21T00:16:52,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1320 sec 2024-11-21T00:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:16:52,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:52,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-21T00:16:52,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:52,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:52,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:52,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:52,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:52,540 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T00:16:52,542 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:52,543 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:52,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:52,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112147053e9a64e2475fa397b9f033112a5f_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:52,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742209_1385 (size=14994) 2024-11-21T00:16:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:52,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:52,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212713, max timestamp=1732148212830Num uniques 859; 2024-11-21T00:16:52,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212713, max timestamp=1732148212830Num uniques 859; 2024-11-21T00:16:52,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148212713, max timestamp=1732148212830Num uniques 859; 2024-11-21T00:16:52,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:52,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:52,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:52,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:53,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112147053e9a64e2475fa397b9f033112a5f_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147053e9a64e2475fa397b9f033112a5f_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:53,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/588a8b4db5b54a0ba5c2049f99cafc06, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/588a8b4db5b54a0ba5c2049f99cafc06 is 175, key is test_row_0/A:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742210_1386 (size=39949) 2024-11-21T00:16:53,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/588a8b4db5b54a0ba5c2049f99cafc06 2024-11-21T00:16:53,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148273040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:53,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/f67475730aa44a4593cc16178d6c30a8 is 50, key is test_row_0/B:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742211_1387 (size=12301) 2024-11-21T00:16:53,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/f67475730aa44a4593cc16178d6c30a8 2024-11-21T00:16:53,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/63055aace8aa480ca1acb2fbe90607d2 is 50, key is test_row_0/C:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742212_1388 (size=12301) 2024-11-21T00:16:53,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/63055aace8aa480ca1acb2fbe90607d2 2024-11-21T00:16:53,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/588a8b4db5b54a0ba5c2049f99cafc06 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06 2024-11-21T00:16:53,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06, entries=200, sequenceid=271, filesize=39.0 K 2024-11-21T00:16:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/f67475730aa44a4593cc16178d6c30a8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8 2024-11-21T00:16:53,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:16:53,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/63055aace8aa480ca1acb2fbe90607d2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2 2024-11-21T00:16:53,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:16:53,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for 719d85aff3bf5e2cc4338f16a05e06b6 in 624ms, sequenceid=271, compaction requested=true 2024-11-21T00:16:53,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:53,163 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:53,164 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:53,165 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:53,165 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:16:53,165 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,165 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/2d316bdd6e474b15ab975207747567eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=36.1 K 2024-11-21T00:16:53,165 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:53,165 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d316bdd6e474b15ab975207747567eb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:53,165 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:16:53,165 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,165 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=100.1 K 2024-11-21T00:16:53,166 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,166 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9045304eaf7f487fa23dc8372235e0f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732148210142 2024-11-21T00:16:53,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06] 2024-11-21T00:16:53,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1718fdbea748467f9d9f27ff4ba9882e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:53,166 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f67475730aa44a4593cc16178d6c30a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212510 2024-11-21T00:16:53,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef0aa7fef1194092bdc7b922a193b6c2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732148210142 2024-11-21T00:16:53,167 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 588a8b4db5b54a0ba5c2049f99cafc06, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212465 2024-11-21T00:16:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:53,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:53,185 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,188 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:53,189 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/726ef9674488446fbe6cf8748330d31e is 50, key is test_row_0/B:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,200 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121df245b8fbca5470684c4b12da8fb621a_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121df245b8fbca5470684c4b12da8fb621a_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121df245b8fbca5470684c4b12da8fb621a_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112109adf7aba714498abd2dcfa8dae05fce_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148213038/Put/seqid=0 2024-11-21T00:16:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742213_1389 (size=12779) 2024-11-21T00:16:53,262 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/726ef9674488446fbe6cf8748330d31e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/726ef9674488446fbe6cf8748330d31e 2024-11-21T00:16:53,268 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into 726ef9674488446fbe6cf8748330d31e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:53,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:53,268 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=13, startTime=1732148213164; duration=0sec 2024-11-21T00:16:53,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:53,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:16:53,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148273261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:53,273 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:53,274 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:16:53,274 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,274 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d7bc8e99829f40e996aa40e838f8dded, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=36.1 K 2024-11-21T00:16:53,274 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d7bc8e99829f40e996aa40e838f8dded, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732148209368 2024-11-21T00:16:53,274 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ad83779d1b33478292fa3866b87d9c92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732148210142 2024-11-21T00:16:53,275 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 63055aace8aa480ca1acb2fbe90607d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212510 2024-11-21T00:16:53,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742214_1390 (size=4469) 2024-11-21T00:16:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742215_1391 (size=14994) 2024-11-21T00:16:53,287 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:53,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112109adf7aba714498abd2dcfa8dae05fce_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112109adf7aba714498abd2dcfa8dae05fce_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:53,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/04df92c14dac4a1d8d64ffe315557d68, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:53,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/04df92c14dac4a1d8d64ffe315557d68 is 175, key is test_row_0/A:col10/1732148213038/Put/seqid=0 2024-11-21T00:16:53,297 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:53,298 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/1b55d22495e04ce48115acebf9a4e7d4 is 50, key is test_row_0/C:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742216_1392 (size=39949) 2024-11-21T00:16:53,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742217_1393 (size=12779) 2024-11-21T00:16:53,354 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/1b55d22495e04ce48115acebf9a4e7d4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/1b55d22495e04ce48115acebf9a4e7d4 2024-11-21T00:16:53,369 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into 1b55d22495e04ce48115acebf9a4e7d4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:53,369 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:53,369 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=13, startTime=1732148213164; duration=0sec 2024-11-21T00:16:53,369 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:53,370 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:16:53,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148273377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:53,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148273596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:53,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,681 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:53,681 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/4e2cd6ba9a50497c82a1666172c848d4 is 175, key is test_row_0/A:col10/1732148212535/Put/seqid=0 2024-11-21T00:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:53,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742218_1394 (size=31733) 2024-11-21T00:16:53,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/04df92c14dac4a1d8d64ffe315557d68 2024-11-21T00:16:53,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/7550cf0967244f878c4a750942897270 is 50, key is test_row_0/B:col10/1732148213038/Put/seqid=0 2024-11-21T00:16:53,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742219_1395 (size=12301) 2024-11-21T00:16:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148273917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:53,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:53,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:54,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,122 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/4e2cd6ba9a50497c82a1666172c848d4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4 2024-11-21T00:16:54,129 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into 4e2cd6ba9a50497c82a1666172c848d4(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:54,129 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:54,129 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=13, startTime=1732148213163; duration=0sec 2024-11-21T00:16:54,130 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:54,130 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:16:54,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/7550cf0967244f878c4a750942897270 2024-11-21T00:16:54,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e6ece172ec3b432ea932d999407eba1a is 50, key is test_row_0/C:col10/1732148213038/Put/seqid=0 2024-11-21T00:16:54,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742220_1396 (size=12301) 2024-11-21T00:16:54,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e6ece172ec3b432ea932d999407eba1a 2024-11-21T00:16:54,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/04df92c14dac4a1d8d64ffe315557d68 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68 2024-11-21T00:16:54,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:54,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:54,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68, entries=200, sequenceid=308, filesize=39.0 K 2024-11-21T00:16:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/7550cf0967244f878c4a750942897270 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270 2024-11-21T00:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T00:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e6ece172ec3b432ea932d999407eba1a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a 2024-11-21T00:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1132ms, sequenceid=308, compaction requested=false 2024-11-21T00:16:54,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,433 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112101aaf9569af845eb9a1f86f9d541a030_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_1/A:col10/1732148213241/Put/seqid=0 2024-11-21T00:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742221_1397 (size=9914) 2024-11-21T00:16:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:54,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148214802, max timestamp=1732148214886Num uniques 859; 2024-11-21T00:16:54,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:54,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148214802, max timestamp=1732148214886Num uniques 859; 2024-11-21T00:16:54,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:54,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148214802, max timestamp=1732148214886Num uniques 859; 2024-11-21T00:16:54,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:16:54,907 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112101aaf9569af845eb9a1f86f9d541a030_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101aaf9569af845eb9a1f86f9d541a030_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/3635ad6a7a074757bab519b2ade2168f, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/3635ad6a7a074757bab519b2ade2168f is 175, key is test_row_1/A:col10/1732148213241/Put/seqid=0 2024-11-21T00:16:54,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742222_1398 (size=22561) 2024-11-21T00:16:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/3635ad6a7a074757bab519b2ade2168f 2024-11-21T00:16:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/17ff01f4f8514d03ac1688598e8cd1e8 is 50, key is test_row_1/B:col10/1732148213241/Put/seqid=0 2024-11-21T00:16:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742223_1399 (size=9857) 2024-11-21T00:16:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148275153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148275287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:55,424 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/17ff01f4f8514d03ac1688598e8cd1e8 2024-11-21T00:16:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f8e86c0bb55f4740b5cb5b3e76116a4e is 50, key is test_row_1/C:col10/1732148213241/Put/seqid=0 2024-11-21T00:16:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742224_1400 (size=9857) 2024-11-21T00:16:55,473 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f8e86c0bb55f4740b5cb5b3e76116a4e 2024-11-21T00:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/3635ad6a7a074757bab519b2ade2168f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f 2024-11-21T00:16:55,486 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f, entries=100, sequenceid=321, filesize=22.0 K 2024-11-21T00:16:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/17ff01f4f8514d03ac1688598e8cd1e8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8 2024-11-21T00:16:55,494 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8, entries=100, sequenceid=321, filesize=9.6 K 2024-11-21T00:16:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/f8e86c0bb55f4740b5cb5b3e76116a4e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e 2024-11-21T00:16:55,509 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e, entries=100, sequenceid=321, filesize=9.6 K 2024-11-21T00:16:55,514 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1082ms, sequenceid=321, compaction requested=true 2024-11-21T00:16:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T00:16:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T00:16:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:55,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:16:55,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T00:16:55,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9810 sec 2024-11-21T00:16:55,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.0020 sec 2024-11-21T00:16:55,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddc09c49279048e2960e195d4d1bb3b0_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742225_1401 (size=14994) 2024-11-21T00:16:55,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:55,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddc09c49279048e2960e195d4d1bb3b0_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddc09c49279048e2960e195d4d1bb3b0_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:55,647 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/0d545ec544c14d3bacd326634d7890d7, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:55,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/0d545ec544c14d3bacd326634d7890d7 is 175, key is test_row_0/A:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148275639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:55,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742226_1402 (size=39949) 2024-11-21T00:16:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148275756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148275978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:56,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/0d545ec544c14d3bacd326634d7890d7 2024-11-21T00:16:56,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8bed08a53fea437db82f335a2c7dde89 is 50, key is test_row_0/B:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:56,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742227_1403 (size=12301) 2024-11-21T00:16:56,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8bed08a53fea437db82f335a2c7dde89 2024-11-21T00:16:56,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/49fdf43b7f644d4cb0e0e57729f25bd1 is 50, key is test_row_0/C:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:56,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742228_1404 (size=12301) 2024-11-21T00:16:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148276296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:56,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/49fdf43b7f644d4cb0e0e57729f25bd1 2024-11-21T00:16:56,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/0d545ec544c14d3bacd326634d7890d7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7 2024-11-21T00:16:56,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7, entries=200, sequenceid=359, filesize=39.0 K 2024-11-21T00:16:56,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8bed08a53fea437db82f335a2c7dde89 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89 2024-11-21T00:16:56,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T00:16:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/49fdf43b7f644d4cb0e0e57729f25bd1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1 2024-11-21T00:16:56,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T00:16:56,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1063ms, sequenceid=359, compaction requested=true 2024-11-21T00:16:56,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:56,601 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:56,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:16:56,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:56,601 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:56,604 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:56,605 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:16:56,605 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,605 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=131.0 K 2024-11-21T00:16:56,605 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,605 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7] 2024-11-21T00:16:56,609 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2cd6ba9a50497c82a1666172c848d4, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212510 2024-11-21T00:16:56,610 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04df92c14dac4a1d8d64ffe315557d68, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148212956 2024-11-21T00:16:56,610 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3635ad6a7a074757bab519b2ade2168f, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148213241 2024-11-21T00:16:56,610 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d545ec544c14d3bacd326634d7890d7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:16:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:16:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:16:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:56,615 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:56,615 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:16:56,615 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,615 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/726ef9674488446fbe6cf8748330d31e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=46.1 K 2024-11-21T00:16:56,617 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 726ef9674488446fbe6cf8748330d31e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212510 2024-11-21T00:16:56,619 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 7550cf0967244f878c4a750942897270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148212956 2024-11-21T00:16:56,620 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ff01f4f8514d03ac1688598e8cd1e8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148213241 2024-11-21T00:16:56,623 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bed08a53fea437db82f335a2c7dde89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:16:56,634 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:56,637 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f94a18945f234d15a3a32c71232e5680_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:56,641 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f94a18945f234d15a3a32c71232e5680_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:56,641 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f94a18945f234d15a3a32c71232e5680_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:56,647 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:56,647 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/ce10d0e30c18434e8391378ab7e32529 is 50, key is test_row_0/B:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:56,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742229_1405 (size=4469) 2024-11-21T00:16:56,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742230_1406 (size=12915) 2024-11-21T00:16:56,695 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/ce10d0e30c18434e8391378ab7e32529 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/ce10d0e30c18434e8391378ab7e32529 2024-11-21T00:16:56,704 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into ce10d0e30c18434e8391378ab7e32529(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:56,704 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:56,704 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=12, startTime=1732148216601; duration=0sec 2024-11-21T00:16:56,704 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:16:56,705 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:16:56,705 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:16:56,707 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:16:56,707 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:16:56,707 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,708 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/1b55d22495e04ce48115acebf9a4e7d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=46.1 K 2024-11-21T00:16:56,708 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b55d22495e04ce48115acebf9a4e7d4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148212510 2024-11-21T00:16:56,712 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ece172ec3b432ea932d999407eba1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148212956 2024-11-21T00:16:56,713 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e86c0bb55f4740b5cb5b3e76116a4e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732148213241 2024-11-21T00:16:56,713 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 49fdf43b7f644d4cb0e0e57729f25bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:16:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:16:56,716 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:16:56,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:16:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T00:16:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:56,723 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:16:56,724 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:16:56,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:16:56,747 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:56,748 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/0dab3481bb404408bee40c62a2d02ecb is 50, key is test_row_0/C:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:56,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742231_1407 (size=12915) 2024-11-21T00:16:56,795 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/0dab3481bb404408bee40c62a2d02ecb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0dab3481bb404408bee40c62a2d02ecb 2024-11-21T00:16:56,806 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into 0dab3481bb404408bee40c62a2d02ecb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:56,806 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:56,806 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=12, startTime=1732148216611; duration=0sec 2024-11-21T00:16:56,806 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:56,806 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:16:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:56,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:16:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:56,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182f9b14cc98a478d9028d2ba3cd2dfdf_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148215637/Put/seqid=0 2024-11-21T00:16:56,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:16:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:56,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742232_1408 (size=14994) 2024-11-21T00:16:56,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:56,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182f9b14cc98a478d9028d2ba3cd2dfdf_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f9b14cc98a478d9028d2ba3cd2dfdf_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:56,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/cddae144cbfe436eb1c6aa727a39b330, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:56,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/cddae144cbfe436eb1c6aa727a39b330 is 175, key is test_row_0/A:col10/1732148215637/Put/seqid=0 2024-11-21T00:16:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742233_1409 (size=39949) 2024-11-21T00:16:56,938 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/cddae144cbfe436eb1c6aa727a39b330 2024-11-21T00:16:56,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9e76c9fa33d64322be58ea247dd0e239 is 50, key is test_row_0/B:col10/1732148215637/Put/seqid=0 2024-11-21T00:16:56,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742234_1410 (size=12301) 2024-11-21T00:16:57,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148277021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:57,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:16:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,083 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#336 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:16:57,083 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/10eeadd5fdc1401fbc121dda325516c6 is 175, key is test_row_0/A:col10/1732148215144/Put/seqid=0 2024-11-21T00:16:57,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742235_1411 (size=31869) 2024-11-21T00:16:57,123 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/10eeadd5fdc1401fbc121dda325516c6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6 2024-11-21T00:16:57,132 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into 10eeadd5fdc1401fbc121dda325516c6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:16:57,132 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:57,132 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=12, startTime=1732148216601; duration=0sec 2024-11-21T00:16:57,132 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:16:57,132 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:16:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148277149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:57,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:57,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:16:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:16:57,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9e76c9fa33d64322be58ea247dd0e239 2024-11-21T00:16:57,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fa3ac1a284734641990498a4abcce003 is 50, key is test_row_0/C:col10/1732148215637/Put/seqid=0 2024-11-21T00:16:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148277367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:57,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742236_1412 (size=12301) 2024-11-21T00:16:57,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fa3ac1a284734641990498a4abcce003 2024-11-21T00:16:57,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/cddae144cbfe436eb1c6aa727a39b330 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330 2024-11-21T00:16:57,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330, entries=200, sequenceid=371, filesize=39.0 K 2024-11-21T00:16:57,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/9e76c9fa33d64322be58ea247dd0e239 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239 2024-11-21T00:16:57,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T00:16:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/fa3ac1a284734641990498a4abcce003 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003 2024-11-21T00:16:57,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T00:16:57,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 719d85aff3bf5e2cc4338f16a05e06b6 in 652ms, sequenceid=371, compaction requested=false 2024-11-21T00:16:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:57,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:16:57,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:57,532 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:16:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:16:57,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b4f69074b215467899854b822c96cd27_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148217001/Put/seqid=0 2024-11-21T00:16:57,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742237_1413 (size=12454) 2024-11-21T00:16:57,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:16:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:57,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148277858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:58,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148277988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:16:58,052 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b4f69074b215467899854b822c96cd27_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b4f69074b215467899854b822c96cd27_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:16:58,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/b261310123a246828da97411b6f0fea3, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:16:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/b261310123a246828da97411b6f0fea3 is 175, key is test_row_0/A:col10/1732148217001/Put/seqid=0 2024-11-21T00:16:58,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742238_1414 (size=31255) 2024-11-21T00:16:58,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148278208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:58,500 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/b261310123a246828da97411b6f0fea3 2024-11-21T00:16:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8282e79c971945b486ab71752a95ae30 is 50, key is test_row_0/B:col10/1732148217001/Put/seqid=0 2024-11-21T00:16:58,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148278538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:58,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742239_1415 (size=12301) 2024-11-21T00:16:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:16:58,980 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8282e79c971945b486ab71752a95ae30 2024-11-21T00:16:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/abcd27e1ac5040eaadd88f383ba0ba5d is 50, key is test_row_0/C:col10/1732148217001/Put/seqid=0 2024-11-21T00:16:59,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742240_1416 (size=12301) 2024-11-21T00:16:59,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:16:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148279057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:16:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:16:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:16:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/abcd27e1ac5040eaadd88f383ba0ba5d 2024-11-21T00:16:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/b261310123a246828da97411b6f0fea3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3 2024-11-21T00:16:59,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3, entries=150, sequenceid=394, filesize=30.5 K 2024-11-21T00:16:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/8282e79c971945b486ab71752a95ae30 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30 2024-11-21T00:16:59,613 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30, entries=150, sequenceid=394, filesize=12.0 K 2024-11-21T00:16:59,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/abcd27e1ac5040eaadd88f383ba0ba5d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d 2024-11-21T00:16:59,658 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-21T00:16:59,662 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 2131ms, sequenceid=394, compaction requested=true 2024-11-21T00:16:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:16:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:16:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T00:16:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T00:16:59,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T00:16:59,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9420 sec 2024-11-21T00:16:59,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.9630 sec 2024-11-21T00:17:00,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:17:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:00,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213d16b4d5f5614b13a8911cb801e8406c_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742241_1417 (size=14994) 2024-11-21T00:17:00,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:00,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213d16b4d5f5614b13a8911cb801e8406c_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d16b4d5f5614b13a8911cb801e8406c_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:00,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/81ab1253a22444ddab542f574267f06f, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:00,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/81ab1253a22444ddab542f574267f06f is 175, key is test_row_0/A:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742242_1418 (size=39949) 2024-11-21T00:17:00,308 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/81ab1253a22444ddab542f574267f06f 2024-11-21T00:17:00,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/4c170ff2a65046b6bf0bba935766f93a is 50, key is test_row_0/B:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742243_1419 (size=12301) 2024-11-21T00:17:00,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/4c170ff2a65046b6bf0bba935766f93a 2024-11-21T00:17:00,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/d1b2c095d1754789b33c936cfbd39ea3 is 50, key is test_row_0/C:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742244_1420 (size=12301) 2024-11-21T00:17:00,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148220329, max timestamp=1732148220392Num uniques 859; 2024-11-21T00:17:00,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148220329, max timestamp=1732148220392Num uniques 859; 2024-11-21T00:17:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:00,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148220329, max timestamp=1732148220392Num uniques 859; 2024-11-21T00:17:00,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:00,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148280595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:00,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148280716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:00,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/d1b2c095d1754789b33c936cfbd39ea3 2024-11-21T00:17:00,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/81ab1253a22444ddab542f574267f06f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f 2024-11-21T00:17:00,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f, entries=200, sequenceid=405, filesize=39.0 K 2024-11-21T00:17:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/4c170ff2a65046b6bf0bba935766f93a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a 2024-11-21T00:17:00,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T00:17:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/d1b2c095d1754789b33c936cfbd39ea3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3 2024-11-21T00:17:00,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T00:17:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 719d85aff3bf5e2cc4338f16a05e06b6 in 697ms, sequenceid=405, compaction requested=true 2024-11-21T00:17:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:00,791 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:00,792 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 719d85aff3bf5e2cc4338f16a05e06b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:00,792 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:00,792 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/A is initiating minor compaction (all files) 2024-11-21T00:17:00,792 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/A in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:00,793 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=139.7 K 2024-11-21T00:17:00,793 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:00,793 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f] 2024-11-21T00:17:00,793 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10eeadd5fdc1401fbc121dda325516c6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:17:00,794 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting cddae144cbfe436eb1c6aa727a39b330, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148215603 2024-11-21T00:17:00,794 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:00,794 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/B is initiating minor compaction (all files) 2024-11-21T00:17:00,794 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/B in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:00,794 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/ce10d0e30c18434e8391378ab7e32529, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=48.7 K 2024-11-21T00:17:00,797 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b261310123a246828da97411b6f0fea3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732148217001 2024-11-21T00:17:00,797 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ce10d0e30c18434e8391378ab7e32529, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:17:00,797 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ab1253a22444ddab542f574267f06f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732148217734 2024-11-21T00:17:00,797 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e76c9fa33d64322be58ea247dd0e239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148215603 2024-11-21T00:17:00,797 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8282e79c971945b486ab71752a95ae30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732148217001 2024-11-21T00:17:00,798 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c170ff2a65046b6bf0bba935766f93a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732148217734 2024-11-21T00:17:00,805 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:00,815 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:17:00,816 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/437e320601be4f8586d86561e73f06fa is 50, key is test_row_0/B:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,817 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411216e9292cb457d495d827b4c2587bbd55c_719d85aff3bf5e2cc4338f16a05e06b6 store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:00,820 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411216e9292cb457d495d827b4c2587bbd55c_719d85aff3bf5e2cc4338f16a05e06b6, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:00,820 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e9292cb457d495d827b4c2587bbd55c_719d85aff3bf5e2cc4338f16a05e06b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:00,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742246_1422 (size=4469) 2024-11-21T00:17:00,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742245_1421 (size=13051) 2024-11-21T00:17:00,843 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#A#compaction#348 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:17:00,843 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/167c1a81667e4f42a8bb4b258ccb0806 is 175, key is test_row_0/A:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742247_1423 (size=32005) 2024-11-21T00:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:17:00,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:00,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T00:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:00,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:00,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:00,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:00,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T00:17:00,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:17:00,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:00,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:17:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:17:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:00,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121592fe2f08237453b96cbff3f4374d332_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148220506/Put/seqid=0 2024-11-21T00:17:00,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742248_1424 (size=14994) 2024-11-21T00:17:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:01,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148281009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148281126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,152 INFO [Thread-1522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1522. 2024-11-21T00:17:01,152 DEBUG [Thread-1522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:01,152 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,152 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:01,158 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-21T00:17:01,158 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:01,158 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,158 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,159 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-21T00:17:01,159 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:01,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:01,159 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,159 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:01,171 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-21T00:17:01,171 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:01,171 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,171 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:01,174 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-21T00:17:01,174 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:01,174 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,174 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:01,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:01,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:01,246 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/437e320601be4f8586d86561e73f06fa as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/437e320601be4f8586d86561e73f06fa 2024-11-21T00:17:01,250 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/B of 719d85aff3bf5e2cc4338f16a05e06b6 into 437e320601be4f8586d86561e73f06fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:01,250 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:01,250 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/B, priority=12, startTime=1732148220792; duration=0sec 2024-11-21T00:17:01,250 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:01,250 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:B 2024-11-21T00:17:01,250 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:01,251 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:01,251 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 719d85aff3bf5e2cc4338f16a05e06b6/C is initiating minor compaction (all files) 2024-11-21T00:17:01,251 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 719d85aff3bf5e2cc4338f16a05e06b6/C in TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,252 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0dab3481bb404408bee40c62a2d02ecb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp, totalSize=48.7 K 2024-11-21T00:17:01,252 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dab3481bb404408bee40c62a2d02ecb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732148215046 2024-11-21T00:17:01,252 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3ac1a284734641990498a4abcce003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148215603 2024-11-21T00:17:01,253 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting abcd27e1ac5040eaadd88f383ba0ba5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732148217001 2024-11-21T00:17:01,253 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b2c095d1754789b33c936cfbd39ea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732148217734 2024-11-21T00:17:01,261 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 719d85aff3bf5e2cc4338f16a05e06b6#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T00:17:01,261 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/7f7d0116d7404a2cbdc144a834cd113a is 50, key is test_row_0/C:col10/1732148217734/Put/seqid=0 2024-11-21T00:17:01,263 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/167c1a81667e4f42a8bb4b258ccb0806 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/167c1a81667e4f42a8bb4b258ccb0806 2024-11-21T00:17:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742249_1425 (size=13051) 2024-11-21T00:17:01,266 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/A of 719d85aff3bf5e2cc4338f16a05e06b6 into 167c1a81667e4f42a8bb4b258ccb0806(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:01,267 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:01,267 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/A, priority=12, startTime=1732148220791; duration=0sec 2024-11-21T00:17:01,267 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:01,267 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:A 2024-11-21T00:17:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46382 deadline: 1732148281317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,317 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18345 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46350 deadline: 1732148281326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,326 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18374 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148281346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:01,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121592fe2f08237453b96cbff3f4374d332_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121592fe2f08237453b96cbff3f4374d332_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:01,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/017e02fd79f9439484f61361c00da2f9, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:01,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/017e02fd79f9439484f61361c00da2f9 is 175, key is test_row_0/A:col10/1732148220506/Put/seqid=0 2024-11-21T00:17:01,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742250_1426 (size=39949) 2024-11-21T00:17:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732148281446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,447 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18358 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46416 deadline: 1732148281457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,457 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in 719d85aff3bf5e2cc4338f16a05e06b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18365 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148281657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:01,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/7f7d0116d7404a2cbdc144a834cd113a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7f7d0116d7404a2cbdc144a834cd113a 2024-11-21T00:17:01,674 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 719d85aff3bf5e2cc4338f16a05e06b6/C of 719d85aff3bf5e2cc4338f16a05e06b6 into 7f7d0116d7404a2cbdc144a834cd113a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:01,674 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:01,674 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., storeName=719d85aff3bf5e2cc4338f16a05e06b6/C, priority=12, startTime=1732148220792; duration=0sec 2024-11-21T00:17:01,674 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:01,674 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 719d85aff3bf5e2cc4338f16a05e06b6:C 2024-11-21T00:17:01,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/017e02fd79f9439484f61361c00da2f9 2024-11-21T00:17:01,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d71008f2eda2431ea8c2b939bcb9c913 is 50, key is test_row_0/B:col10/1732148220506/Put/seqid=0 2024-11-21T00:17:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742251_1427 (size=12301) 2024-11-21T00:17:01,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:01,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:02,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 343 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732148282166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:02,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d71008f2eda2431ea8c2b939bcb9c913 2024-11-21T00:17:02,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/3c2ccd526744466ab4c8dd8a5bddf980 is 50, key is test_row_0/C:col10/1732148220506/Put/seqid=0 2024-11-21T00:17:02,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742252_1428 (size=12301) 2024-11-21T00:17:02,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/3c2ccd526744466ab4c8dd8a5bddf980 2024-11-21T00:17:02,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/017e02fd79f9439484f61361c00da2f9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/017e02fd79f9439484f61361c00da2f9 2024-11-21T00:17:02,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:02,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:02,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/017e02fd79f9439484f61361c00da2f9, entries=200, sequenceid=442, filesize=39.0 K 2024-11-21T00:17:02,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/d71008f2eda2431ea8c2b939bcb9c913 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d71008f2eda2431ea8c2b939bcb9c913 2024-11-21T00:17:02,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d71008f2eda2431ea8c2b939bcb9c913, entries=150, sequenceid=442, filesize=12.0 K 2024-11-21T00:17:02,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/3c2ccd526744466ab4c8dd8a5bddf980 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/3c2ccd526744466ab4c8dd8a5bddf980 2024-11-21T00:17:02,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/3c2ccd526744466ab4c8dd8a5bddf980, entries=150, sequenceid=442, filesize=12.0 K 2024-11-21T00:17:02,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1666ms, sequenceid=442, compaction requested=false 2024-11-21T00:17:02,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:02,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T00:17:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:17:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b03e01539cc4e909a7fc7261aa4e5f1_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148220985/Put/seqid=0 2024-11-21T00:17:02,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742253_1429 (size=12454) 2024-11-21T00:17:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:02,782 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b03e01539cc4e909a7fc7261aa4e5f1_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b03e01539cc4e909a7fc7261aa4e5f1_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:02,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/169864f8ea464d70a6519fb338196228, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/169864f8ea464d70a6519fb338196228 is 175, key is test_row_0/A:col10/1732148220985/Put/seqid=0 2024-11-21T00:17:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742254_1430 (size=31255) 2024-11-21T00:17:02,806 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/169864f8ea464d70a6519fb338196228 2024-11-21T00:17:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/98c2d4f70a0943be97e0c6053c2851af is 50, key is test_row_0/B:col10/1732148220985/Put/seqid=0 2024-11-21T00:17:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742255_1431 (size=12301) 2024-11-21T00:17:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:03,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. as already flushing 2024-11-21T00:17:03,179 INFO [Thread-1511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1511. 2024-11-21T00:17:03,179 DEBUG [Thread-1511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:03,179 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:03,179 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:03,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:03,251 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/98c2d4f70a0943be97e0c6053c2851af 2024-11-21T00:17:03,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/7645f133f21f45968cd90e41060e0a51 is 50, key is test_row_0/C:col10/1732148220985/Put/seqid=0 2024-11-21T00:17:03,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742256_1432 (size=12301) 2024-11-21T00:17:03,675 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/7645f133f21f45968cd90e41060e0a51 2024-11-21T00:17:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/169864f8ea464d70a6519fb338196228 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/169864f8ea464d70a6519fb338196228 2024-11-21T00:17:03,693 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/169864f8ea464d70a6519fb338196228, entries=150, sequenceid=455, filesize=30.5 K 2024-11-21T00:17:03,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/98c2d4f70a0943be97e0c6053c2851af as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/98c2d4f70a0943be97e0c6053c2851af 2024-11-21T00:17:03,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/98c2d4f70a0943be97e0c6053c2851af, entries=150, sequenceid=455, filesize=12.0 K 2024-11-21T00:17:03,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/7645f133f21f45968cd90e41060e0a51 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7645f133f21f45968cd90e41060e0a51 2024-11-21T00:17:03,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7645f133f21f45968cd90e41060e0a51, entries=150, sequenceid=455, filesize=12.0 K 2024-11-21T00:17:03,717 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 719d85aff3bf5e2cc4338f16a05e06b6 in 973ms, sequenceid=455, compaction requested=true 2024-11-21T00:17:03,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 719d85aff3bf5e2cc4338f16a05e06b6: 2024-11-21T00:17:03,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:03,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T00:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T00:17:03,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T00:17:03,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8190 sec 2024-11-21T00:17:03,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.8250 sec 2024-11-21T00:17:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:17:05,073 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:11,378 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-21T00:17:11,378 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:11,378 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,378 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:11,378 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-11-21T00:17:11,378 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:11,378 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,378 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:11,458 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-11-21T00:17:11,458 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:11,458 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,458 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:11,537 INFO [Thread-1517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1517. 2024-11-21T00:17:11,537 DEBUG [Thread-1517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:11,538 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,538 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 288 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 733 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2199 rows 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 744 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2225 rows 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 747 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2188 rows 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 758 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2204 rows 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 736 2024-11-21T00:17:11,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2208 rows 2024-11-21T00:17:11,538 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:17:11,538 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:17:11,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:11,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:11,539 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:17:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:11,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148231542"}]},"ts":"1732148231542"} 2024-11-21T00:17:11,544 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:17:11,544 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:17:11,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:17:11,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, UNASSIGN}] 2024-11-21T00:17:11,546 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, UNASSIGN 2024-11-21T00:17:11,547 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:11,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, UNASSIGN because future has completed 2024-11-21T00:17:11,548 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:17:11,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:11,700 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:11,700 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:17:11,700 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 719d85aff3bf5e2cc4338f16a05e06b6, disabling compactions & flushes 2024-11-21T00:17:11,700 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:11,700 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:11,700 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. after waiting 0 ms 2024-11-21T00:17:11,700 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:11,701 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 719d85aff3bf5e2cc4338f16a05e06b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=A 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=B 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 719d85aff3bf5e2cc4338f16a05e06b6, store=C 2024-11-21T00:17:11,701 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:11,709 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112125ef4103af4a4ee3b6a0785ab0191391_719d85aff3bf5e2cc4338f16a05e06b6 is 50, key is test_row_0/A:col10/1732148231376/Put/seqid=0 2024-11-21T00:17:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742257_1433 (size=12454) 2024-11-21T00:17:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:12,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:17:12,120 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:12,133 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112125ef4103af4a4ee3b6a0785ab0191391_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112125ef4103af4a4ee3b6a0785ab0191391_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:12,134 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/5fd0c2a6020a4a438326004bc0e2caef, store: [table=TestAcidGuarantees family=A region=719d85aff3bf5e2cc4338f16a05e06b6] 2024-11-21T00:17:12,134 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/5fd0c2a6020a4a438326004bc0e2caef is 175, key is test_row_0/A:col10/1732148231376/Put/seqid=0 2024-11-21T00:17:12,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742258_1434 (size=31255) 2024-11-21T00:17:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:12,553 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/5fd0c2a6020a4a438326004bc0e2caef 2024-11-21T00:17:12,559 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/126c5e2f7a0c4fa0b8e884f1819e9134 is 50, key is test_row_0/B:col10/1732148231376/Put/seqid=0 2024-11-21T00:17:12,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742259_1435 (size=12301) 2024-11-21T00:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:12,963 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/126c5e2f7a0c4fa0b8e884f1819e9134 2024-11-21T00:17:12,969 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e8f1d1e304214c12bd60109c432f9a33 is 50, key is test_row_0/C:col10/1732148231376/Put/seqid=0 2024-11-21T00:17:12,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742260_1436 (size=12301) 2024-11-21T00:17:13,373 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e8f1d1e304214c12bd60109c432f9a33 2024-11-21T00:17:13,381 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/A/5fd0c2a6020a4a438326004bc0e2caef as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/5fd0c2a6020a4a438326004bc0e2caef 2024-11-21T00:17:13,386 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/5fd0c2a6020a4a438326004bc0e2caef, entries=150, sequenceid=463, filesize=30.5 K 2024-11-21T00:17:13,386 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/B/126c5e2f7a0c4fa0b8e884f1819e9134 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/126c5e2f7a0c4fa0b8e884f1819e9134 2024-11-21T00:17:13,391 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/126c5e2f7a0c4fa0b8e884f1819e9134, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T00:17:13,392 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/.tmp/C/e8f1d1e304214c12bd60109c432f9a33 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e8f1d1e304214c12bd60109c432f9a33 2024-11-21T00:17:13,396 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e8f1d1e304214c12bd60109c432f9a33, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T00:17:13,397 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1697ms, sequenceid=463, compaction requested=true 2024-11-21T00:17:13,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f] to archive 2024-11-21T00:17:13,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:13,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7cfcf3ab5b1c44abab1d75dfca1b8731 2024-11-21T00:17:13,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d37ac67ffbd249a082b213f610db1328 2024-11-21T00:17:13,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/91c477e9345b411189ed4ab5ca8ba499 2024-11-21T00:17:13,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/edc69770e4a24d908207a6d9bd079741 2024-11-21T00:17:13,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/18518efc49304025b3f10f06b5bfa1d1 2024-11-21T00:17:13,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/a474a1546c184796abec63e210bfad21 2024-11-21T00:17:13,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7c982b83155a49c893cb1050d1871c28 2024-11-21T00:17:13,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/d0b362a0ebba469a93a4e8cf21163508 2024-11-21T00:17:13,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/e7ff00f747064a748bef1703ba45a43b 2024-11-21T00:17:13,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/7de067dcabbb49e7a51da09973badd6f 2024-11-21T00:17:13,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/f3934c2f72ec4fc08de35f9ae827db8e 2024-11-21T00:17:13,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/1718fdbea748467f9d9f27ff4ba9882e 2024-11-21T00:17:13,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/ef0aa7fef1194092bdc7b922a193b6c2 2024-11-21T00:17:13,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/588a8b4db5b54a0ba5c2049f99cafc06 2024-11-21T00:17:13,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/4e2cd6ba9a50497c82a1666172c848d4 2024-11-21T00:17:13,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/04df92c14dac4a1d8d64ffe315557d68 2024-11-21T00:17:13,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/3635ad6a7a074757bab519b2ade2168f 2024-11-21T00:17:13,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/0d545ec544c14d3bacd326634d7890d7 2024-11-21T00:17:13,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/10eeadd5fdc1401fbc121dda325516c6 2024-11-21T00:17:13,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/cddae144cbfe436eb1c6aa727a39b330 2024-11-21T00:17:13,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/b261310123a246828da97411b6f0fea3 2024-11-21T00:17:13,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/81ab1253a22444ddab542f574267f06f 2024-11-21T00:17:13,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b38cc86f4c5e4c5f83d84d50d155f23b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/209b4852f0ac4f2699f2f4f111a35163, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/2d316bdd6e474b15ab975207747567eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/726ef9674488446fbe6cf8748330d31e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/ce10d0e30c18434e8391378ab7e32529, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a] to archive 2024-11-21T00:17:13,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:13,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b1bb5369016148a5af99ce7303265900 2024-11-21T00:17:13,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/234b9f84175845fdbe2dfc72bc6139eb 2024-11-21T00:17:13,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d7e0e3402ae942868a672e2f1087b1af 2024-11-21T00:17:13,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b38cc86f4c5e4c5f83d84d50d155f23b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b38cc86f4c5e4c5f83d84d50d155f23b 2024-11-21T00:17:13,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/89f9fa9b8cc64e908be45df6aba4c7cb 2024-11-21T00:17:13,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/23b5a8fd66684ba5bff454bf5270007d 2024-11-21T00:17:13,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/cc14b587fb6146e3916a64a1d995d506 2024-11-21T00:17:13,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/209b4852f0ac4f2699f2f4f111a35163 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/209b4852f0ac4f2699f2f4f111a35163 2024-11-21T00:17:13,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/62e631ecbace4a7bb30d323f16a40fcc 2024-11-21T00:17:13,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/a89b5de19694486c9e1b14d241382e7d 2024-11-21T00:17:13,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/2d316bdd6e474b15ab975207747567eb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/2d316bdd6e474b15ab975207747567eb 2024-11-21T00:17:13,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/b3d1ca980f394827a010d014706c5c37 2024-11-21T00:17:13,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9045304eaf7f487fa23dc8372235e0f9 2024-11-21T00:17:13,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/726ef9674488446fbe6cf8748330d31e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/726ef9674488446fbe6cf8748330d31e 2024-11-21T00:17:13,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/f67475730aa44a4593cc16178d6c30a8 2024-11-21T00:17:13,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/7550cf0967244f878c4a750942897270 2024-11-21T00:17:13,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/17ff01f4f8514d03ac1688598e8cd1e8 2024-11-21T00:17:13,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/ce10d0e30c18434e8391378ab7e32529 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/ce10d0e30c18434e8391378ab7e32529 2024-11-21T00:17:13,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8bed08a53fea437db82f335a2c7dde89 2024-11-21T00:17:13,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/9e76c9fa33d64322be58ea247dd0e239 2024-11-21T00:17:13,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/8282e79c971945b486ab71752a95ae30 2024-11-21T00:17:13,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/4c170ff2a65046b6bf0bba935766f93a 2024-11-21T00:17:13,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/09549b18df5e4baabed99b627200dd0c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b0af1f0bf531434a9e7caa5edb5a17ad, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d7bc8e99829f40e996aa40e838f8dded, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/1b55d22495e04ce48115acebf9a4e7d4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0dab3481bb404408bee40c62a2d02ecb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3] to archive 2024-11-21T00:17:13,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:13,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0ca14bea31c143bf9550c84b6417b09f 2024-11-21T00:17:13,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fc86caddfd804a59963fe395f09b4b41 2024-11-21T00:17:13,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/c860e348c4cc4f01aea58636b37edf69 2024-11-21T00:17:13,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/09549b18df5e4baabed99b627200dd0c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/09549b18df5e4baabed99b627200dd0c 2024-11-21T00:17:13,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f19e1efe72a940fe92629662a454077d 2024-11-21T00:17:13,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/6e3655c11657401dbece11f58fe6fa72 2024-11-21T00:17:13,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b81b35940b9649d58d3a47fddd781f44 2024-11-21T00:17:13,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b0af1f0bf531434a9e7caa5edb5a17ad to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/b0af1f0bf531434a9e7caa5edb5a17ad 2024-11-21T00:17:13,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4ce86762d22847cf9cbac253d6f5d1c4 2024-11-21T00:17:13,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/4fb353ada5444035be95dc84e7552ac7 2024-11-21T00:17:13,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d7bc8e99829f40e996aa40e838f8dded to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d7bc8e99829f40e996aa40e838f8dded 2024-11-21T00:17:13,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/a8198e05ef14448da3523b7a860db4f6 2024-11-21T00:17:13,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/ad83779d1b33478292fa3866b87d9c92 2024-11-21T00:17:13,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/1b55d22495e04ce48115acebf9a4e7d4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/1b55d22495e04ce48115acebf9a4e7d4 2024-11-21T00:17:13,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/63055aace8aa480ca1acb2fbe90607d2 2024-11-21T00:17:13,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e6ece172ec3b432ea932d999407eba1a 2024-11-21T00:17:13,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/f8e86c0bb55f4740b5cb5b3e76116a4e 2024-11-21T00:17:13,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0dab3481bb404408bee40c62a2d02ecb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/0dab3481bb404408bee40c62a2d02ecb 2024-11-21T00:17:13,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/49fdf43b7f644d4cb0e0e57729f25bd1 2024-11-21T00:17:13,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/fa3ac1a284734641990498a4abcce003 2024-11-21T00:17:13,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/abcd27e1ac5040eaadd88f383ba0ba5d 2024-11-21T00:17:13,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/d1b2c095d1754789b33c936cfbd39ea3 2024-11-21T00:17:13,469 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=4 2024-11-21T00:17:13,470 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. 2024-11-21T00:17:13,471 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 719d85aff3bf5e2cc4338f16a05e06b6: Waiting for close lock at 1732148231700Running coprocessor pre-close hooks at 1732148231700Disabling compacts and flushes for region at 1732148231700Disabling writes for close at 1732148231700Obtaining lock to block concurrent updates at 1732148231701 (+1 ms)Preparing flush snapshotting stores in 719d85aff3bf5e2cc4338f16a05e06b6 at 1732148231701Finished memstore snapshotting TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732148231701Flushing stores of TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6. at 1732148231702 (+1 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/A: creating writer at 1732148231702Flushing mob file 719d85aff3bf5e2cc4338f16a05e06b6/A: appending metadata at 1732148231708 (+6 ms)Flushing mob file 719d85aff3bf5e2cc4338f16a05e06b6/A: closing flushed file at 1732148231708Flushing 719d85aff3bf5e2cc4338f16a05e06b6/A: appending metadata at 1732148232134 (+426 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/A: closing flushed file at 1732148232134Flushing 719d85aff3bf5e2cc4338f16a05e06b6/B: creating writer at 1732148232556 (+422 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/B: appending metadata at 1732148232559 (+3 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/B: closing flushed file at 1732148232559Flushing 719d85aff3bf5e2cc4338f16a05e06b6/C: creating writer at 1732148232966 (+407 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/C: appending metadata at 1732148232969 (+3 ms)Flushing 719d85aff3bf5e2cc4338f16a05e06b6/C: closing flushed file at 1732148232969Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5995db4: reopening flushed file at 1732148233380 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b3bf3d2: reopening flushed file at 1732148233386 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a5a6d44: reopening flushed file at 1732148233391 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 719d85aff3bf5e2cc4338f16a05e06b6 in 1697ms, sequenceid=463, compaction requested=true at 1732148233397 (+6 ms)Writing region close event to WAL at 1732148233466 (+69 ms)Running coprocessor post-close hooks at 1732148233470 (+4 ms)Closed at 1732148233470 2024-11-21T00:17:13,473 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,474 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=719d85aff3bf5e2cc4338f16a05e06b6, regionState=CLOSED 2024-11-21T00:17:13,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:13,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T00:17:13,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 719d85aff3bf5e2cc4338f16a05e06b6, server=5ed4808ef0e6,40705,1732148113864 in 1.9280 sec 2024-11-21T00:17:13,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-21T00:17:13,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=719d85aff3bf5e2cc4338f16a05e06b6, UNASSIGN in 1.9310 sec 2024-11-21T00:17:13,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T00:17:13,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9350 sec 2024-11-21T00:17:13,482 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148233482"}]},"ts":"1732148233482"} 2024-11-21T00:17:13,483 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:17:13,483 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:17:13,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9440 sec 2024-11-21T00:17:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T00:17:13,698 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:13,699 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:17:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,701 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T00:17:13,702 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits] 2024-11-21T00:17:13,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/017e02fd79f9439484f61361c00da2f9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/017e02fd79f9439484f61361c00da2f9 2024-11-21T00:17:13,710 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/167c1a81667e4f42a8bb4b258ccb0806 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/167c1a81667e4f42a8bb4b258ccb0806 2024-11-21T00:17:13,711 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/169864f8ea464d70a6519fb338196228 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/169864f8ea464d70a6519fb338196228 2024-11-21T00:17:13,712 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/5fd0c2a6020a4a438326004bc0e2caef to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/A/5fd0c2a6020a4a438326004bc0e2caef 2024-11-21T00:17:13,714 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/126c5e2f7a0c4fa0b8e884f1819e9134 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/126c5e2f7a0c4fa0b8e884f1819e9134 2024-11-21T00:17:13,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/437e320601be4f8586d86561e73f06fa to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/437e320601be4f8586d86561e73f06fa 2024-11-21T00:17:13,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/98c2d4f70a0943be97e0c6053c2851af to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/98c2d4f70a0943be97e0c6053c2851af 2024-11-21T00:17:13,716 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d71008f2eda2431ea8c2b939bcb9c913 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/B/d71008f2eda2431ea8c2b939bcb9c913 2024-11-21T00:17:13,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/3c2ccd526744466ab4c8dd8a5bddf980 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/3c2ccd526744466ab4c8dd8a5bddf980 2024-11-21T00:17:13,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7645f133f21f45968cd90e41060e0a51 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7645f133f21f45968cd90e41060e0a51 2024-11-21T00:17:13,719 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7f7d0116d7404a2cbdc144a834cd113a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/7f7d0116d7404a2cbdc144a834cd113a 2024-11-21T00:17:13,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e8f1d1e304214c12bd60109c432f9a33 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/C/e8f1d1e304214c12bd60109c432f9a33 2024-11-21T00:17:13,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits/466.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6/recovered.edits/466.seqid 2024-11-21T00:17:13,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,722 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:17:13,723 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:17:13,723 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:17:13,727 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101aaf9569af845eb9a1f86f9d541a030_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101aaf9569af845eb9a1f86f9d541a030_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,730 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112109adf7aba714498abd2dcfa8dae05fce_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112109adf7aba714498abd2dcfa8dae05fce_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,733 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112125ef4103af4a4ee3b6a0785ab0191391_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112125ef4103af4a4ee3b6a0785ab0191391_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,735 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fae0eabc0fb4661a77960f8c15c34a9_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fae0eabc0fb4661a77960f8c15c34a9_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,736 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121330062e5bf5140289e83f3337fe3616c_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121330062e5bf5140289e83f3337fe3616c_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,737 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b03e01539cc4e909a7fc7261aa4e5f1_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b03e01539cc4e909a7fc7261aa4e5f1_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,740 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d16b4d5f5614b13a8911cb801e8406c_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d16b4d5f5614b13a8911cb801e8406c_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,741 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213e0fcec86116471fa8975a130e5286a0_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213e0fcec86116471fa8975a130e5286a0_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,751 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112146ae7c47d71443e39fd1221408e970b8_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112146ae7c47d71443e39fd1221408e970b8_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147053e9a64e2475fa397b9f033112a5f_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147053e9a64e2475fa397b9f033112a5f_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,755 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121592fe2f08237453b96cbff3f4374d332_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121592fe2f08237453b96cbff3f4374d332_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,756 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159e051070c56440ca70fba44f6801f63_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112159e051070c56440ca70fba44f6801f63_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215bfa6ffc3aa54c4ab197bf190ae57755_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215bfa6ffc3aa54c4ab197bf190ae57755_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,758 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f9b14cc98a478d9028d2ba3cd2dfdf_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f9b14cc98a478d9028d2ba3cd2dfdf_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218738cb19b15445aba3d177af59d415b3_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218738cb19b15445aba3d177af59d415b3_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,760 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121937a4d72be7b448992e52571b17cfbbe_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121937a4d72be7b448992e52571b17cfbbe_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b002554d090d4ff093c724c361b06495_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b002554d090d4ff093c724c361b06495_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b4f69074b215467899854b822c96cd27_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b4f69074b215467899854b822c96cd27_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc8ba20974214b2aa2b7c1d863426239_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc8ba20974214b2aa2b7c1d863426239_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,763 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddc09c49279048e2960e195d4d1bb3b0_719d85aff3bf5e2cc4338f16a05e06b6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddc09c49279048e2960e195d4d1bb3b0_719d85aff3bf5e2cc4338f16a05e06b6 2024-11-21T00:17:13,764 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:17:13,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,767 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:17:13,769 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:17:13,770 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,770 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:17:13,770 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148233770"}]},"ts":"9223372036854775807"} 2024-11-21T00:17:13,773 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:17:13,773 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 719d85aff3bf5e2cc4338f16a05e06b6, NAME => 'TestAcidGuarantees,,1732148198102.719d85aff3bf5e2cc4338f16a05e06b6.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:17:13,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:17:13,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148233773"}]},"ts":"9223372036854775807"} 2024-11-21T00:17:13,776 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:17:13,776 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-21T00:17:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T00:17:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:17:13,807 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:13,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 252), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=947 (was 930) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=241 (was 1957) 2024-11-21T00:17:13,832 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=947, ProcessCount=11, AvailableMemoryMB=240 2024-11-21T00:17:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:17:13,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:13,836 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:17:13,836 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:13,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-21T00:17:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:13,837 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:17:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742261_1437 (size=1061) 2024-11-21T00:17:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:14,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:17:14,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742262_1438 (size=53) 2024-11-21T00:17:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6f23b76c1c42e5c3a6984684bda2f10b, disabling compactions & flushes 2024-11-21T00:17:14,650 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. after waiting 0 ms 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,650 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f23b76c1c42e5c3a6984684bda2f10b: Waiting for close lock at 1732148234650Disabling compacts and flushes for region at 1732148234650Disabling writes for close at 1732148234650Writing region close event to WAL at 1732148234650Closed at 1732148234650 2024-11-21T00:17:14,651 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:17:14,652 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148234651"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148234651"}]},"ts":"1732148234651"} 2024-11-21T00:17:14,653 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:17:14,654 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:17:14,655 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148234654"}]},"ts":"1732148234654"} 2024-11-21T00:17:14,656 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:17:14,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, ASSIGN}] 2024-11-21T00:17:14,658 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, ASSIGN 2024-11-21T00:17:14,659 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:17:14,810 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6f23b76c1c42e5c3a6984684bda2f10b, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:14,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, ASSIGN because future has completed 2024-11-21T00:17:14,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:14,967 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,967 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:17:14,968 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,968 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:17:14,968 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,968 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,970 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,971 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:14,971 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f23b76c1c42e5c3a6984684bda2f10b columnFamilyName A 2024-11-21T00:17:14,971 DEBUG [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:14,971 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(327): Store=6f23b76c1c42e5c3a6984684bda2f10b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:14,972 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,973 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:14,973 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f23b76c1c42e5c3a6984684bda2f10b columnFamilyName B 2024-11-21T00:17:14,973 DEBUG [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:14,973 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(327): Store=6f23b76c1c42e5c3a6984684bda2f10b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:14,973 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,974 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:14,974 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f23b76c1c42e5c3a6984684bda2f10b columnFamilyName C 2024-11-21T00:17:14,974 DEBUG [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:14,975 INFO [StoreOpener-6f23b76c1c42e5c3a6984684bda2f10b-1 {}] regionserver.HStore(327): Store=6f23b76c1c42e5c3a6984684bda2f10b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:14,975 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,975 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,976 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,976 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:14,980 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,980 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,981 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:17:14,982 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,988 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:17:14,989 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 6f23b76c1c42e5c3a6984684bda2f10b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66578627, jitterRate=-0.0079011470079422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:17:14,989 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:14,990 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 6f23b76c1c42e5c3a6984684bda2f10b: Running coprocessor pre-open hook at 1732148234969Writing region info on filesystem at 1732148234969Initializing all the Stores at 1732148234969Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148234969Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148234969Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148234969Cleaning up temporary data from old regions at 1732148234980 (+11 ms)Running coprocessor post-open hooks at 1732148234989 (+9 ms)Region opened successfully at 1732148234990 (+1 ms) 2024-11-21T00:17:14,991 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., pid=116, masterSystemTime=1732148234964 2024-11-21T00:17:14,993 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,993 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:14,994 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6f23b76c1c42e5c3a6984684bda2f10b, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:14,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:14,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T00:17:14,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 in 184 msec 2024-11-21T00:17:15,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-21T00:17:15,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, ASSIGN in 341 msec 2024-11-21T00:17:15,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:17:15,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148235006"}]},"ts":"1732148235006"} 2024-11-21T00:17:15,007 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:17:15,008 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:17:15,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-21T00:17:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-21T00:17:15,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:15,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:15,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:15,988 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16724f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:15,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38935a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,991 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:15,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:15,993 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b992dfe 2024-11-21T00:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:17:15,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:17:15,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:15,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:15,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6936539d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:15,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb830ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:15,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a02aa11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547250f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,004 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453c5685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,005 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4391c0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae23ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279d8e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae1d50d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faa1c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df14cfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e53108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48995759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae147f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,024 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82b3ab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd71c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,029 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33de88ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,032 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d27770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:16,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:16,036 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f483af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:16,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f29ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:16,040 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,041 DEBUG [Thread-1965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,041 DEBUG [Thread-1969 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,042 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,042 DEBUG [Thread-1967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,043 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,046 DEBUG [Thread-1973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-21T00:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,051 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:17:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,053 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:16,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:16,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:16,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:16,064 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,078 DEBUG [Thread-1971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,080 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:16,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2] 2024-11-21T00:17:16,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a3a3934ba0e643ec9deef117e9ad4ea9 is 50, key is test_row_0/A:col10/1732148236061/Put/seqid=0 2024-11-21T00:17:16,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742263_1439 (size=16681) 2024-11-21T00:17:16,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148236080, max timestamp=1732148236117Num uniques 859; 2024-11-21T00:17:16,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:16,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148236080, max timestamp=1732148236116Num uniques 859; 2024-11-21T00:17:16,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:16,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a3a3934ba0e643ec9deef117e9ad4ea9 2024-11-21T00:17:16,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148236080, max timestamp=1732148236116Num uniques 859; 2024-11-21T00:17:16,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:17:16,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/08c8281337e8418db5ae2bca7d04229d is 50, key is test_row_0/B:col10/1732148236061/Put/seqid=0 2024-11-21T00:17:16,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742264_1440 (size=12001) 2024-11-21T00:17:16,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/08c8281337e8418db5ae2bca7d04229d 2024-11-21T00:17:16,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:17:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148296195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148296205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148296207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148296211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148296203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/ff2b5e9635d94fcea54c4700e4ba880c is 50, key is test_row_0/C:col10/1732148236061/Put/seqid=0 2024-11-21T00:17:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742265_1441 (size=12001) 2024-11-21T00:17:16,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148296326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148296339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148296340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148296340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148296346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:17:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:17:16,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:17:16,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148296547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148296560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148296560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148296567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148296567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:17:16,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:16,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:17:16,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/ff2b5e9635d94fcea54c4700e4ba880c 2024-11-21T00:17:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a3a3934ba0e643ec9deef117e9ad4ea9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9 2024-11-21T00:17:16,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9, entries=250, sequenceid=11, filesize=16.3 K 2024-11-21T00:17:16,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/08c8281337e8418db5ae2bca7d04229d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d 2024-11-21T00:17:16,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d, entries=150, sequenceid=11, filesize=11.7 K 2024-11-21T00:17:16,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/ff2b5e9635d94fcea54c4700e4ba880c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c 2024-11-21T00:17:16,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c, entries=150, sequenceid=11, filesize=11.7 K 2024-11-21T00:17:16,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6f23b76c1c42e5c3a6984684bda2f10b in 682ms, sequenceid=11, compaction requested=false 2024-11-21T00:17:16,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:16,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,830 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:16,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d410ba20a01443ba9d95c3734982eecf is 50, key is test_row_0/A:col10/1732148236205/Put/seqid=0 2024-11-21T00:17:16,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742266_1442 (size=12001) 2024-11-21T00:17:16,866 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d410ba20a01443ba9d95c3734982eecf 2024-11-21T00:17:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/c7be262cd9004e93acb15e3c3a9639ba is 50, key is test_row_0/B:col10/1732148236205/Put/seqid=0 2024-11-21T00:17:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:16,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742267_1443 (size=12001) 2024-11-21T00:17:16,895 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/c7be262cd9004e93acb15e3c3a9639ba 2024-11-21T00:17:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148296894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148296895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/46385c0e4363479390bbf06de8916d00 is 50, key is test_row_0/C:col10/1732148236205/Put/seqid=0 2024-11-21T00:17:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148296897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148296898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148296898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:16,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742268_1444 (size=12001) 2024-11-21T00:17:16,922 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/46385c0e4363479390bbf06de8916d00 2024-11-21T00:17:16,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d410ba20a01443ba9d95c3734982eecf as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf 2024-11-21T00:17:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T00:17:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/c7be262cd9004e93acb15e3c3a9639ba as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba 2024-11-21T00:17:16,950 DEBUG [regionserver/5ed4808ef0e6:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-21T00:17:16,952 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T00:17:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/46385c0e4363479390bbf06de8916d00 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00 2024-11-21T00:17:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T00:17:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6f23b76c1c42e5c3a6984684bda2f10b in 128ms, sequenceid=47, compaction requested=false 2024-11-21T00:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-21T00:17:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-21T00:17:16,962 WARN [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-21T00:17:16,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T00:17:16,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 908 msec 2024-11-21T00:17:16,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 915 msec 2024-11-21T00:17:17,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:17:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:17,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:17,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0ccdcb4f16554365ae37e1c835e3a1ad is 50, key is test_row_0/A:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:17,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742269_1445 (size=12001) 2024-11-21T00:17:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148297062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148297063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148297076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148297078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148297085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148297189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148297189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:17:17,196 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:17,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-21T00:17:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:17,199 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:17,199 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:17,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148297196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148297196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148297209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:17,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148297396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148297398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148297417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148297417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148297426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0ccdcb4f16554365ae37e1c835e3a1ad 2024-11-21T00:17:17,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9b86de7fac9243b9ba3dcb87a0a8640f is 50, key is test_row_0/B:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:17,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742270_1446 (size=12001) 2024-11-21T00:17:17,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:17,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:17,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148297717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148297717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148297737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148297737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148297746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:17,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9b86de7fac9243b9ba3dcb87a0a8640f 2024-11-21T00:17:17,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1564610af7b54390851c95ba8c75a36e is 50, key is test_row_0/C:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:17,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742271_1447 (size=12001) 2024-11-21T00:17:17,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1564610af7b54390851c95ba8c75a36e 2024-11-21T00:17:17,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:17,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0ccdcb4f16554365ae37e1c835e3a1ad as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad 2024-11-21T00:17:18,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T00:17:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9b86de7fac9243b9ba3dcb87a0a8640f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f 2024-11-21T00:17:18,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T00:17:18,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1564610af7b54390851c95ba8c75a36e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e 2024-11-21T00:17:18,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T00:17:18,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6f23b76c1c42e5c3a6984684bda2f10b in 1047ms, sequenceid=59, compaction requested=true 2024-11-21T00:17:18,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:18,061 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:18,061 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:18,067 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:18,067 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:18,067 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:18,067 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=39.7 K 2024-11-21T00:17:18,068 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:18,068 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:18,068 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:18,068 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.2 K 2024-11-21T00:17:18,072 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 08c8281337e8418db5ae2bca7d04229d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732148236057 2024-11-21T00:17:18,073 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a3934ba0e643ec9deef117e9ad4ea9, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732148236055 2024-11-21T00:17:18,076 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c7be262cd9004e93acb15e3c3a9639ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732148236186 2024-11-21T00:17:18,076 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d410ba20a01443ba9d95c3734982eecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732148236186 2024-11-21T00:17:18,079 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ccdcb4f16554365ae37e1c835e3a1ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:18,079 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b86de7fac9243b9ba3dcb87a0a8640f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:18,106 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:18,107 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a2b26bbdc2d24634aaa10186f511e4a8 is 50, key is test_row_0/A:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:18,143 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:18,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:17:18,146 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4f726a266b7b4bc5ba61c9efd17cb3d3 is 50, key is test_row_0/B:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:18,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:18,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742272_1448 (size=12104) 2024-11-21T00:17:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bc8a6ec9b27b45b99b678e3afbbfef25 is 50, key is test_row_0/A:col10/1732148237067/Put/seqid=0 2024-11-21T00:17:18,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742273_1449 (size=12104) 2024-11-21T00:17:18,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742274_1450 (size=12001) 2024-11-21T00:17:18,222 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bc8a6ec9b27b45b99b678e3afbbfef25 2024-11-21T00:17:18,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e6c53d585061489490478e6e96531dab is 50, key is test_row_0/B:col10/1732148237067/Put/seqid=0 2024-11-21T00:17:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742275_1451 (size=12001) 2024-11-21T00:17:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e6c53d585061489490478e6e96531dab 2024-11-21T00:17:18,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148298287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148298287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/cef1079c676f4291932cb9c6dde07c2d is 50, key is test_row_0/C:col10/1732148237067/Put/seqid=0 2024-11-21T00:17:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148298293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148298307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742276_1452 (size=12001) 2024-11-21T00:17:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148298324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:18,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148298417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148298417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148298417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148298426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148298437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,564 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a2b26bbdc2d24634aaa10186f511e4a8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a2b26bbdc2d24634aaa10186f511e4a8 2024-11-21T00:17:18,573 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into a2b26bbdc2d24634aaa10186f511e4a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:18,573 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:18,573 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148238061; duration=0sec 2024-11-21T00:17:18,573 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:18,573 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:18,573 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:18,576 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:18,576 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:18,577 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:18,577 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.2 K 2024-11-21T00:17:18,577 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff2b5e9635d94fcea54c4700e4ba880c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732148236057 2024-11-21T00:17:18,577 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46385c0e4363479390bbf06de8916d00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732148236186 2024-11-21T00:17:18,577 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1564610af7b54390851c95ba8c75a36e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:18,583 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:18,584 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/babaae6ba1ac4a919723b48f8e66bb97 is 50, key is test_row_0/C:col10/1732148237011/Put/seqid=0 2024-11-21T00:17:18,588 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4f726a266b7b4bc5ba61c9efd17cb3d3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4f726a266b7b4bc5ba61c9efd17cb3d3 2024-11-21T00:17:18,595 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into 4f726a266b7b4bc5ba61c9efd17cb3d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:18,595 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:18,595 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148238061; duration=0sec 2024-11-21T00:17:18,595 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:18,595 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:18,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742277_1453 (size=12104) 2024-11-21T00:17:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148298636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148298636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148298637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148298637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148298656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:18,732 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/cef1079c676f4291932cb9c6dde07c2d 2024-11-21T00:17:18,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bc8a6ec9b27b45b99b678e3afbbfef25 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25 2024-11-21T00:17:18,760 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T00:17:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e6c53d585061489490478e6e96531dab as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab 2024-11-21T00:17:18,765 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T00:17:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/cef1079c676f4291932cb9c6dde07c2d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d 2024-11-21T00:17:18,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T00:17:18,773 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 6f23b76c1c42e5c3a6984684bda2f10b in 625ms, sequenceid=79, compaction requested=false 2024-11-21T00:17:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-21T00:17:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-21T00:17:18,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T00:17:18,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-21T00:17:18,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5800 sec 2024-11-21T00:17:18,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:18,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/176e7a99f1a44b1187052373f4d197db is 50, key is test_row_0/A:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742278_1454 (size=14341) 2024-11-21T00:17:18,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/176e7a99f1a44b1187052373f4d197db 2024-11-21T00:17:18,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/8ae66b8f164a43558d14e9f803a79d49 is 50, key is test_row_0/B:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:18,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742279_1455 (size=12001) 2024-11-21T00:17:18,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/8ae66b8f164a43558d14e9f803a79d49 2024-11-21T00:17:18,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/61d1d8b3a55244299506a162867074a9 is 50, key is test_row_0/C:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:18,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742280_1456 (size=12001) 2024-11-21T00:17:19,003 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/babaae6ba1ac4a919723b48f8e66bb97 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/babaae6ba1ac4a919723b48f8e66bb97 2024-11-21T00:17:19,007 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into babaae6ba1ac4a919723b48f8e66bb97(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:19,007 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:19,007 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148238061; duration=0sec 2024-11-21T00:17:19,007 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:19,007 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:19,037 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:17:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148299043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148299047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148299047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148299047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148299054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148299166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148299166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148299166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148299168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148299177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:17:19,347 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:19,347 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-21T00:17:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:17:19,350 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:19,350 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:19,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:19,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/61d1d8b3a55244299506a162867074a9 2024-11-21T00:17:19,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148299386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148299387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148299387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148299387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/176e7a99f1a44b1187052373f4d197db as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db 2024-11-21T00:17:19,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db, entries=200, sequenceid=95, filesize=14.0 K 2024-11-21T00:17:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/8ae66b8f164a43558d14e9f803a79d49 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49 2024-11-21T00:17:19,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148299396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T00:17:19,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/61d1d8b3a55244299506a162867074a9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9 2024-11-21T00:17:19,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T00:17:19,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6f23b76c1c42e5c3a6984684bda2f10b in 474ms, sequenceid=95, compaction requested=true 2024-11-21T00:17:19,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:19,428 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:19,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:19,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:19,429 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:19,429 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:19,429 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:19,430 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:19,430 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a2b26bbdc2d24634aaa10186f511e4a8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=37.5 K 2024-11-21T00:17:19,430 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:19,430 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2b26bbdc2d24634aaa10186f511e4a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:19,430 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:19,430 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:19,430 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4f726a266b7b4bc5ba61c9efd17cb3d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.3 K 2024-11-21T00:17:19,431 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8a6ec9b27b45b99b678e3afbbfef25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148237060 2024-11-21T00:17:19,431 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f726a266b7b4bc5ba61c9efd17cb3d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:19,431 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176e7a99f1a44b1187052373f4d197db, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:19,431 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c53d585061489490478e6e96531dab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148237060 2024-11-21T00:17:19,431 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae66b8f164a43558d14e9f803a79d49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:19,443 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:19,443 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0fbea2e9cf664845b43a318fb8080db7 is 50, key is test_row_0/A:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:19,454 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:19,455 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/48f2cde2768e4dec870fd245665c877d is 50, key is test_row_0/B:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:17:19,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742281_1457 (size=12207) 2024-11-21T00:17:19,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742282_1458 (size=12207) 2024-11-21T00:17:19,472 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/48f2cde2768e4dec870fd245665c877d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/48f2cde2768e4dec870fd245665c877d 2024-11-21T00:17:19,478 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into 48f2cde2768e4dec870fd245665c877d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:19,478 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:19,478 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148239429; duration=0sec 2024-11-21T00:17:19,478 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:19,478 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:19,478 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:19,479 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:19,479 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:19,479 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:19,479 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/babaae6ba1ac4a919723b48f8e66bb97, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.3 K 2024-11-21T00:17:19,480 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting babaae6ba1ac4a919723b48f8e66bb97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732148236896 2024-11-21T00:17:19,480 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cef1079c676f4291932cb9c6dde07c2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148237060 2024-11-21T00:17:19,480 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d1d8b3a55244299506a162867074a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:19,494 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:19,495 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/dd89c2b31c644935b0ca612e5cbd779b is 50, key is test_row_0/C:col10/1732148238266/Put/seqid=0 2024-11-21T00:17:19,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T00:17:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:19,512 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:17:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bd37349bd4d64c5882e68de9f7ef125a is 50, key is test_row_0/A:col10/1732148239044/Put/seqid=0 2024-11-21T00:17:19,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742283_1459 (size=12207) 2024-11-21T00:17:19,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742284_1460 (size=12001) 2024-11-21T00:17:19,564 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bd37349bd4d64c5882e68de9f7ef125a 2024-11-21T00:17:19,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5bb8557b8bfc4d758a6c68a034a5dd65 is 50, key is test_row_0/B:col10/1732148239044/Put/seqid=0 2024-11-21T00:17:19,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742285_1461 (size=12001) 2024-11-21T00:17:19,603 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5bb8557b8bfc4d758a6c68a034a5dd65 2024-11-21T00:17:19,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/197726ed3fcd48289240d838ae530d81 is 50, key is test_row_0/C:col10/1732148239044/Put/seqid=0 2024-11-21T00:17:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742286_1462 (size=12001) 2024-11-21T00:17:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:17:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:19,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:19,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148299719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148299721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148299725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148299732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148299738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148299847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148299847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148299847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148299847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,867 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0fbea2e9cf664845b43a318fb8080db7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0fbea2e9cf664845b43a318fb8080db7 2024-11-21T00:17:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148299856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:19,875 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 0fbea2e9cf664845b43a318fb8080db7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:19,875 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:19,875 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148239428; duration=0sec 2024-11-21T00:17:19,875 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:19,875 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:19,959 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/dd89c2b31c644935b0ca612e5cbd779b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/dd89c2b31c644935b0ca612e5cbd779b 2024-11-21T00:17:19,968 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into dd89c2b31c644935b0ca612e5cbd779b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:19,968 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:19,968 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148239429; duration=0sec 2024-11-21T00:17:19,968 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:19,968 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:17:20,059 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/197726ed3fcd48289240d838ae530d81 2024-11-21T00:17:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148300056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148300056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148300056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148300057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bd37349bd4d64c5882e68de9f7ef125a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a 2024-11-21T00:17:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,073 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:17:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5bb8557b8bfc4d758a6c68a034a5dd65 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65 2024-11-21T00:17:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148300076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,085 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:17:20,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/197726ed3fcd48289240d838ae530d81 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81 2024-11-21T00:17:20,093 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T00:17:20,102 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6f23b76c1c42e5c3a6984684bda2f10b in 581ms, sequenceid=116, compaction requested=false 2024-11-21T00:17:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-21T00:17:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-21T00:17:20,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T00:17:20,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 753 msec 2024-11-21T00:17:20,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 758 msec 2024-11-21T00:17:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:20,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:20,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cc5855c1db8543308bbbc33a386ae65c is 50, key is test_row_0/A:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742287_1463 (size=14391) 2024-11-21T00:17:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148300407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148300408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148300419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148300421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148300422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:17:20,487 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:20,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T00:17:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:20,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:20,490 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:20,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148300526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148300526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148300538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148300538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148300547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:20,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:20,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:20,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148300746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148300748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148300760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148300762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cc5855c1db8543308bbbc33a386ae65c 2024-11-21T00:17:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148300767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:20,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9ff929712ea94e328d3c8d91df6f7f28 is 50, key is test_row_0/B:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:20,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742288_1464 (size=12051) 2024-11-21T00:17:20,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:20,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148301066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148301066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148301087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148301087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148301086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:21,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9ff929712ea94e328d3c8d91df6f7f28 2024-11-21T00:17:21,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/71ee95019a094a25b5f75c91a63894af is 50, key is test_row_0/C:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:21,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:21,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742289_1465 (size=12051) 2024-11-21T00:17:21,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:21,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148301599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148301603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148301607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148301616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148301627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:21,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/71ee95019a094a25b5f75c91a63894af 2024-11-21T00:17:21,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cc5855c1db8543308bbbc33a386ae65c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c 2024-11-21T00:17:21,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c, entries=200, sequenceid=132, filesize=14.1 K 2024-11-21T00:17:21,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9ff929712ea94e328d3c8d91df6f7f28 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28 2024-11-21T00:17:21,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T00:17:21,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/71ee95019a094a25b5f75c91a63894af as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af 2024-11-21T00:17:21,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T00:17:21,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6f23b76c1c42e5c3a6984684bda2f10b in 1339ms, sequenceid=132, compaction requested=true 2024-11-21T00:17:21,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:21,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:21,707 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:21,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:21,708 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:21,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:21,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:21,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:21,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:21,708 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:21,708 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:21,708 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,709 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/48f2cde2768e4dec870fd245665c877d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.4 K 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:21,709 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,709 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0fbea2e9cf664845b43a318fb8080db7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=37.7 K 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f2cde2768e4dec870fd245665c877d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fbea2e9cf664845b43a318fb8080db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb8557b8bfc4d758a6c68a034a5dd65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148239044 2024-11-21T00:17:21,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd37349bd4d64c5882e68de9f7ef125a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148239044 2024-11-21T00:17:21,710 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff929712ea94e328d3c8d91df6f7f28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239721 2024-11-21T00:17:21,710 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5855c1db8543308bbbc33a386ae65c, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239715 2024-11-21T00:17:21,718 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:21,719 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/5614126d4cfb49fb9d205fd4bf1fea71 is 50, key is test_row_0/A:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:21,721 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:21,722 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/d49f50e1bfb641b99036adbf7d4b722b is 50, key is test_row_0/B:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:21,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:17:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,738 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:17:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:21,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:21,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:21,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a09a048f9c35403196a0e702e12eda96 is 50, key is test_row_0/A:col10/1732148240419/Put/seqid=0 2024-11-21T00:17:21,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742290_1466 (size=12359) 2024-11-21T00:17:21,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742291_1467 (size=12359) 2024-11-21T00:17:21,776 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/d49f50e1bfb641b99036adbf7d4b722b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/d49f50e1bfb641b99036adbf7d4b722b 2024-11-21T00:17:21,781 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into d49f50e1bfb641b99036adbf7d4b722b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:21,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:21,781 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148241707; duration=0sec 2024-11-21T00:17:21,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:21,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:21,781 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:21,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:21,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:21,783 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:21,783 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/dd89c2b31c644935b0ca612e5cbd779b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.4 K 2024-11-21T00:17:21,783 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting dd89c2b31c644935b0ca612e5cbd779b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732148238266 2024-11-21T00:17:21,784 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 197726ed3fcd48289240d838ae530d81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732148239044 2024-11-21T00:17:21,784 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ee95019a094a25b5f75c91a63894af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239721 2024-11-21T00:17:21,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742292_1468 (size=12151) 2024-11-21T00:17:21,794 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:21,795 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/710eaf4896c844ce9effbb411179fea4 is 50, key is test_row_0/C:col10/1732148240366/Put/seqid=0 2024-11-21T00:17:21,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742293_1469 (size=12359) 2024-11-21T00:17:22,162 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/5614126d4cfb49fb9d205fd4bf1fea71 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5614126d4cfb49fb9d205fd4bf1fea71 2024-11-21T00:17:22,166 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 5614126d4cfb49fb9d205fd4bf1fea71(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:22,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:22,166 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148241707; duration=0sec 2024-11-21T00:17:22,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:22,166 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:22,194 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a09a048f9c35403196a0e702e12eda96 2024-11-21T00:17:22,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/74495f56497e4a99bd19da6c104b9e2c is 50, key is test_row_0/B:col10/1732148240419/Put/seqid=0 2024-11-21T00:17:22,219 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/710eaf4896c844ce9effbb411179fea4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/710eaf4896c844ce9effbb411179fea4 2024-11-21T00:17:22,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742294_1470 (size=12151) 2024-11-21T00:17:22,227 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into 710eaf4896c844ce9effbb411179fea4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:22,227 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:22,227 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148241708; duration=0sec 2024-11-21T00:17:22,227 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:22,227 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:22,228 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/74495f56497e4a99bd19da6c104b9e2c 2024-11-21T00:17:22,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aabd6f36c7f24ad486ac50558fe50827 is 50, key is test_row_0/C:col10/1732148240419/Put/seqid=0 2024-11-21T00:17:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742295_1471 (size=12151) 2024-11-21T00:17:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:22,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:22,642 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aabd6f36c7f24ad486ac50558fe50827 2024-11-21T00:17:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a09a048f9c35403196a0e702e12eda96 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96 2024-11-21T00:17:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148302661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148302658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,679 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:17:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/74495f56497e4a99bd19da6c104b9e2c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c 2024-11-21T00:17:22,687 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:17:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aabd6f36c7f24ad486ac50558fe50827 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827 2024-11-21T00:17:22,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148302673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148302675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148302676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,701 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T00:17:22,702 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 6f23b76c1c42e5c3a6984684bda2f10b in 964ms, sequenceid=150, compaction requested=false 2024-11-21T00:17:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T00:17:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T00:17:22,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T00:17:22,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-21T00:17:22,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2200 sec 2024-11-21T00:17:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:22,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:22,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a24dd4e2172543068a80a7afe5ab4bc3 is 50, key is test_row_0/A:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:22,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742296_1472 (size=14541) 2024-11-21T00:17:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148302813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148302817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148302818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148302819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148302820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148302936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148302937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148302937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148302937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148302937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:22,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:22,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148303157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148303157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148303157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148303157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148303157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a24dd4e2172543068a80a7afe5ab4bc3 2024-11-21T00:17:23,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a446f40ae3884d6ca1bc988c135abc9a is 50, key is test_row_0/B:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:23,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742297_1473 (size=12151) 2024-11-21T00:17:23,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a446f40ae3884d6ca1bc988c135abc9a 2024-11-21T00:17:23,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/9a4d948bce114454bea3b728f7369518 is 50, key is test_row_0/C:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:23,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742298_1474 (size=12151) 2024-11-21T00:17:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148303476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148303476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148303477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148303478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148303478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:23,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/9a4d948bce114454bea3b728f7369518 2024-11-21T00:17:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a24dd4e2172543068a80a7afe5ab4bc3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3 2024-11-21T00:17:23,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3, entries=200, sequenceid=169, filesize=14.2 K 2024-11-21T00:17:23,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a446f40ae3884d6ca1bc988c135abc9a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a 2024-11-21T00:17:23,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T00:17:23,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/9a4d948bce114454bea3b728f7369518 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518 2024-11-21T00:17:23,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T00:17:23,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6f23b76c1c42e5c3a6984684bda2f10b in 866ms, sequenceid=169, compaction requested=true 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:23,660 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:23,660 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:23,660 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:23,660 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:23,661 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:23,661 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:23,661 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/d49f50e1bfb641b99036adbf7d4b722b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.8 K 2024-11-21T00:17:23,661 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5614126d4cfb49fb9d205fd4bf1fea71, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=38.1 K 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5614126d4cfb49fb9d205fd4bf1fea71, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239721 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d49f50e1bfb641b99036adbf7d4b722b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239721 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 74495f56497e4a99bd19da6c104b9e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148240403 2024-11-21T00:17:23,661 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09a048f9c35403196a0e702e12eda96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148240403 2024-11-21T00:17:23,662 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a446f40ae3884d6ca1bc988c135abc9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:23,662 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24dd4e2172543068a80a7afe5ab4bc3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:23,674 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:23,675 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/1e9798f1b596467ca11f1ccb51e8e78a is 50, key is test_row_0/A:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:23,678 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:23,679 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9c35ba9d82ff41f5807f2b7d1cf1070f is 50, key is test_row_0/B:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:23,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742299_1475 (size=12561) 2024-11-21T00:17:23,698 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/1e9798f1b596467ca11f1ccb51e8e78a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1e9798f1b596467ca11f1ccb51e8e78a 2024-11-21T00:17:23,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742300_1476 (size=12561) 2024-11-21T00:17:23,706 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 1e9798f1b596467ca11f1ccb51e8e78a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:23,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:23,706 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148243660; duration=0sec 2024-11-21T00:17:23,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:23,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:23,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:23,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:23,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:23,708 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:23,708 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/710eaf4896c844ce9effbb411179fea4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=35.8 K 2024-11-21T00:17:23,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710eaf4896c844ce9effbb411179fea4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732148239721 2024-11-21T00:17:23,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting aabd6f36c7f24ad486ac50558fe50827, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732148240403 2024-11-21T00:17:23,709 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a4d948bce114454bea3b728f7369518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:23,717 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/9c35ba9d82ff41f5807f2b7d1cf1070f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9c35ba9d82ff41f5807f2b7d1cf1070f 2024-11-21T00:17:23,728 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:23,729 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/bb120ee228134dc781b3a0942f916a92 is 50, key is test_row_0/C:col10/1732148242792/Put/seqid=0 2024-11-21T00:17:23,731 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into 9c35ba9d82ff41f5807f2b7d1cf1070f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:23,731 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:23,731 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148243660; duration=0sec 2024-11-21T00:17:23,731 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:23,731 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:23,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742301_1477 (size=12561) 2024-11-21T00:17:23,751 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/bb120ee228134dc781b3a0942f916a92 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/bb120ee228134dc781b3a0942f916a92 2024-11-21T00:17:23,756 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into bb120ee228134dc781b3a0942f916a92(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:23,756 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:23,756 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148243660; duration=0sec 2024-11-21T00:17:23,756 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:23,756 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:23,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:24,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/546c1a32274548088985e60d8968fa30 is 50, key is test_row_0/A:col10/1732148243990/Put/seqid=0 2024-11-21T00:17:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742302_1478 (size=12151) 2024-11-21T00:17:24,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/546c1a32274548088985e60d8968fa30 2024-11-21T00:17:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/b05f1c9741164496aa70071426dfa357 is 50, key is test_row_0/B:col10/1732148243990/Put/seqid=0 2024-11-21T00:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148304015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148304017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148304018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148304020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742303_1479 (size=12151) 2024-11-21T00:17:24,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/b05f1c9741164496aa70071426dfa357 2024-11-21T00:17:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148304034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1095e0ef5475489cb86b6b05d6a6c388 is 50, key is test_row_0/C:col10/1732148243990/Put/seqid=0 2024-11-21T00:17:24,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742304_1480 (size=12151) 2024-11-21T00:17:24,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1095e0ef5475489cb86b6b05d6a6c388 2024-11-21T00:17:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/546c1a32274548088985e60d8968fa30 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30 2024-11-21T00:17:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T00:17:24,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/b05f1c9741164496aa70071426dfa357 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357 2024-11-21T00:17:24,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T00:17:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/1095e0ef5475489cb86b6b05d6a6c388 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388 2024-11-21T00:17:24,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T00:17:24,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6f23b76c1c42e5c3a6984684bda2f10b in 103ms, sequenceid=189, compaction requested=false 2024-11-21T00:17:24,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:24,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T00:17:24,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:24,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a388895ff8ab43d5a9ab4ccc7406c1a2 is 50, key is test_row_0/A:col10/1732148244018/Put/seqid=0 2024-11-21T00:17:24,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742305_1481 (size=14541) 2024-11-21T00:17:24,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a388895ff8ab43d5a9ab4ccc7406c1a2 2024-11-21T00:17:24,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/1a89edb77f084764bfde01f254435dbc is 50, key is test_row_0/B:col10/1732148244018/Put/seqid=0 2024-11-21T00:17:24,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742306_1482 (size=12151) 2024-11-21T00:17:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148304159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148304162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148304163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148304164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148304165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148304276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148304276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148304287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148304287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148304288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148304486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148304487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148304509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148304509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148304509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/1a89edb77f084764bfde01f254435dbc 2024-11-21T00:17:24,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f48e65b95bc944fdacc8dc0b361db225 is 50, key is test_row_0/C:col10/1732148244018/Put/seqid=0 2024-11-21T00:17:24,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742307_1483 (size=12151) 2024-11-21T00:17:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:17:24,652 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:24,653 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T00:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:17:24,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:24,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:24,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:17:24,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148304806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148304806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148304827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:17:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:24,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:24,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148304828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148304828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:17:24,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:17:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:24,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:25,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f48e65b95bc944fdacc8dc0b361db225 2024-11-21T00:17:25,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/a388895ff8ab43d5a9ab4ccc7406c1a2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2 2024-11-21T00:17:25,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2, entries=200, sequenceid=205, filesize=14.2 K 2024-11-21T00:17:25,081 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T00:17:25,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/1a89edb77f084764bfde01f254435dbc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc 2024-11-21T00:17:25,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T00:17:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f48e65b95bc944fdacc8dc0b361db225 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225 2024-11-21T00:17:25,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T00:17:25,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6f23b76c1c42e5c3a6984684bda2f10b in 963ms, sequenceid=205, compaction requested=true 2024-11-21T00:17:25,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:25,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:25,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:25,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:25,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:17:25,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:25,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T00:17:25,108 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:25,108 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:25,111 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:25,112 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:25,112 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,112 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1e9798f1b596467ca11f1ccb51e8e78a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=38.3 K 2024-11-21T00:17:25,117 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9798f1b596467ca11f1ccb51e8e78a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:25,120 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:25,120 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546c1a32274548088985e60d8968fa30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732148242817 2024-11-21T00:17:25,121 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:25,121 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,121 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/bb120ee228134dc781b3a0942f916a92, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.0 K 2024-11-21T00:17:25,126 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting bb120ee228134dc781b3a0942f916a92, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:25,126 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a388895ff8ab43d5a9ab4ccc7406c1a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732148244013 2024-11-21T00:17:25,131 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1095e0ef5475489cb86b6b05d6a6c388, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732148242817 2024-11-21T00:17:25,132 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f48e65b95bc944fdacc8dc0b361db225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732148244013 2024-11-21T00:17:25,145 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:25,145 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/cf0afeb8c5e9418c8b5d4006b56d91a2 is 50, key is test_row_0/C:col10/1732148244018/Put/seqid=0 2024-11-21T00:17:25,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:17:25,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,150 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:25,151 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/de85bcd286ad47f5943fa595e9a9bf74 is 50, key is test_row_0/A:col10/1732148244018/Put/seqid=0 2024-11-21T00:17:25,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742308_1484 (size=12663) 2024-11-21T00:17:25,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/56286baf8c0941c8b28b17c59db88e8b is 50, key is test_row_0/A:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:25,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742309_1485 (size=12663) 2024-11-21T00:17:25,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742310_1486 (size=12151) 2024-11-21T00:17:25,212 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/56286baf8c0941c8b28b17c59db88e8b 2024-11-21T00:17:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5f671a12bc144567baf69b3779a0d05e is 50, key is test_row_0/B:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:25,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742311_1487 (size=12151) 2024-11-21T00:17:25,280 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5f671a12bc144567baf69b3779a0d05e 2024-11-21T00:17:25,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/e5d2bdcebfa146d4ba5bc55e7db0059f is 50, key is test_row_0/C:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:17:25,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742312_1488 (size=12151) 2024-11-21T00:17:25,324 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/e5d2bdcebfa146d4ba5bc55e7db0059f 2024-11-21T00:17:25,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:25,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/56286baf8c0941c8b28b17c59db88e8b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b 2024-11-21T00:17:25,336 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T00:17:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/5f671a12bc144567baf69b3779a0d05e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e 2024-11-21T00:17:25,341 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T00:17:25,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/e5d2bdcebfa146d4ba5bc55e7db0059f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f 2024-11-21T00:17:25,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T00:17:25,346 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 6f23b76c1c42e5c3a6984684bda2f10b in 198ms, sequenceid=221, compaction requested=true 2024-11-21T00:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T00:17:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T00:17:25,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T00:17:25,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 667 msec 2024-11-21T00:17:25,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:25,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 697 msec 2024-11-21T00:17:25,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bb65922c969f42839c4aba7bc2fa58e6 is 50, key is test_row_1/A:col10/1732148245352/Put/seqid=0 2024-11-21T00:17:25,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742313_1489 (size=12147) 2024-11-21T00:17:25,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148305417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148245401, max timestamp=1732148245433Num uniques 859; 2024-11-21T00:17:25,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:17:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148245401, max timestamp=1732148245433Num uniques 859; 2024-11-21T00:17:25,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:17:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148245401, max timestamp=1732148245433Num uniques 859; 2024-11-21T00:17:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:17:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148305497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148305505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148305513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148305514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148305546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,594 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/cf0afeb8c5e9418c8b5d4006b56d91a2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cf0afeb8c5e9418c8b5d4006b56d91a2 2024-11-21T00:17:25,598 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into cf0afeb8c5e9418c8b5d4006b56d91a2(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:25,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:25,598 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148245103; duration=0sec 2024-11-21T00:17:25,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:25,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:25,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:25,599 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:25,599 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:25,599 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,599 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9c35ba9d82ff41f5807f2b7d1cf1070f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=47.9 K 2024-11-21T00:17:25,599 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c35ba9d82ff41f5807f2b7d1cf1070f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148242644 2024-11-21T00:17:25,600 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b05f1c9741164496aa70071426dfa357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732148242817 2024-11-21T00:17:25,600 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a89edb77f084764bfde01f254435dbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732148244013 2024-11-21T00:17:25,602 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f671a12bc144567baf69b3779a0d05e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732148244154 2024-11-21T00:17:25,606 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/de85bcd286ad47f5943fa595e9a9bf74 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/de85bcd286ad47f5943fa595e9a9bf74 2024-11-21T00:17:25,613 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into de85bcd286ad47f5943fa595e9a9bf74(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:25,613 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:25,614 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148245102; duration=0sec 2024-11-21T00:17:25,614 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:25,614 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:25,631 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:25,631 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/52d92cd308cf4fbc851e9b7845cfe62f is 50, key is test_row_0/B:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:25,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148305626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148305637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148305637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742314_1490 (size=12697) 2024-11-21T00:17:25,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148305646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,666 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/52d92cd308cf4fbc851e9b7845cfe62f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/52d92cd308cf4fbc851e9b7845cfe62f 2024-11-21T00:17:25,669 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into 52d92cd308cf4fbc851e9b7845cfe62f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:25,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:25,669 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=12, startTime=1732148245102; duration=0sec 2024-11-21T00:17:25,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:25,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148305767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bb65922c969f42839c4aba7bc2fa58e6 2024-11-21T00:17:25,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/26ffa45153c94dceb0861981f452ba7b is 50, key is test_row_1/B:col10/1732148245352/Put/seqid=0 2024-11-21T00:17:25,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742315_1491 (size=9757) 2024-11-21T00:17:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:17:25,812 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:25,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T00:17:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:25,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:25,821 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:25,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148305846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148305856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148305856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148305866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:25,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:25,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148306087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148306166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148306177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148306177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148306186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/26ffa45153c94dceb0861981f452ba7b 2024-11-21T00:17:26,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/114197629c574859b420d4b6f1dd7cac is 50, key is test_row_1/C:col10/1732148245352/Put/seqid=0 2024-11-21T00:17:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742316_1492 (size=9757) 2024-11-21T00:17:26,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:26,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148306596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/114197629c574859b420d4b6f1dd7cac 2024-11-21T00:17:26,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/bb65922c969f42839c4aba7bc2fa58e6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6 2024-11-21T00:17:26,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T00:17:26,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/26ffa45153c94dceb0861981f452ba7b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b 2024-11-21T00:17:26,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b, entries=100, sequenceid=231, filesize=9.5 K 2024-11-21T00:17:26,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/114197629c574859b420d4b6f1dd7cac as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac 2024-11-21T00:17:26,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac, entries=100, sequenceid=231, filesize=9.5 K 2024-11-21T00:17:26,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.00 KB/136187 for 6f23b76c1c42e5c3a6984684bda2f10b in 1274ms, sequenceid=231, compaction requested=true 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:26,628 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:26,628 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:26,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:26,629 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:17:26,629 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:17:26,629 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:26,629 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:26,629 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:26,630 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:26,630 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:26,630 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,630 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/de85bcd286ad47f5943fa595e9a9bf74, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.1 K 2024-11-21T00:17:26,631 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting de85bcd286ad47f5943fa595e9a9bf74, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732148244013 2024-11-21T00:17:26,631 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:26,631 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56286baf8c0941c8b28b17c59db88e8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732148244154 2024-11-21T00:17:26,631 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:26,631 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,631 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cf0afeb8c5e9418c8b5d4006b56d91a2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=33.8 K 2024-11-21T00:17:26,631 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb65922c969f42839c4aba7bc2fa58e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732148245347 2024-11-21T00:17:26,632 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0afeb8c5e9418c8b5d4006b56d91a2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732148244013 2024-11-21T00:17:26,632 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d2bdcebfa146d4ba5bc55e7db0059f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732148244154 2024-11-21T00:17:26,633 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 114197629c574859b420d4b6f1dd7cac, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732148245347 2024-11-21T00:17:26,638 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:26,639 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/5633aa96578546b9bdbf805dbe923b2e is 50, key is test_row_0/A:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:26,642 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:26,642 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f1de50dcd20c4127a428faf51c686b12 is 50, key is test_row_0/C:col10/1732148244164/Put/seqid=0 2024-11-21T00:17:26,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742317_1493 (size=12765) 2024-11-21T00:17:26,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742318_1494 (size=12765) 2024-11-21T00:17:26,653 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/5633aa96578546b9bdbf805dbe923b2e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5633aa96578546b9bdbf805dbe923b2e 2024-11-21T00:17:26,664 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f1de50dcd20c4127a428faf51c686b12 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f1de50dcd20c4127a428faf51c686b12 2024-11-21T00:17:26,665 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 5633aa96578546b9bdbf805dbe923b2e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:26,665 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:26,665 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148246628; duration=0sec 2024-11-21T00:17:26,665 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:26,665 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:26,669 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into f1de50dcd20c4127a428faf51c686b12(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:26,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:26,669 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148246628; duration=0sec 2024-11-21T00:17:26,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:26,669 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148245481, max timestamp=1732148246697Num uniques 1009; 2024-11-21T00:17:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148245481, max timestamp=1732148246697Num uniques 1009; 2024-11-21T00:17:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-21T00:17:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-21T00:17:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:26,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=64.46 KB heapSize=177.88 KB 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:26,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/7242b6f8cb63469b945f3dc07e01275b is 50, key is test_row_0/A:col10/1732148245510/Put/seqid=0 2024-11-21T00:17:26,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742319_1495 (size=14741) 2024-11-21T00:17:26,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148306793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148306810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148306811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148306814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:26,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:26,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:26,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148306926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148306926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148306937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148306937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:27,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/7242b6f8cb63469b945f3dc07e01275b 2024-11-21T00:17:27,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4c0fade723714e099ed33d6376f2cbbc is 50, key is test_row_0/B:col10/1732148245510/Put/seqid=0 2024-11-21T00:17:27,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742320_1496 (size=12301) 2024-11-21T00:17:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148307146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148307146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148307156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148307156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148307467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148307467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148307476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148307476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4c0fade723714e099ed33d6376f2cbbc 2024-11-21T00:17:27,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/67f40c2e58814b779db1dd4f8eadec95 is 50, key is test_row_0/C:col10/1732148245510/Put/seqid=0 2024-11-21T00:17:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742321_1497 (size=12301) 2024-11-21T00:17:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148307617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:27,627 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:27,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:27,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/67f40c2e58814b779db1dd4f8eadec95 2024-11-21T00:17:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/7242b6f8cb63469b945f3dc07e01275b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b 2024-11-21T00:17:27,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b, entries=200, sequenceid=274, filesize=14.4 K 2024-11-21T00:17:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4c0fade723714e099ed33d6376f2cbbc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc 2024-11-21T00:17:27,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc, entries=150, sequenceid=274, filesize=12.0 K 2024-11-21T00:17:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/67f40c2e58814b779db1dd4f8eadec95 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95 2024-11-21T00:17:27,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95, entries=150, sequenceid=274, filesize=12.0 K 2024-11-21T00:17:27,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for 6f23b76c1c42e5c3a6984684bda2f10b in 1242ms, sequenceid=274, compaction requested=true 2024-11-21T00:17:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:27,945 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:17:27,945 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:27,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:27,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:27,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:27,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:27,946 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:27,947 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:27,947 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,947 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/52d92cd308cf4fbc851e9b7845cfe62f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=33.9 K 2024-11-21T00:17:27,947 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d92cd308cf4fbc851e9b7845cfe62f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732148244154 2024-11-21T00:17:27,947 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ffa45153c94dceb0861981f452ba7b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732148245347 2024-11-21T00:17:27,947 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0fade723714e099ed33d6376f2cbbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732148245481 2024-11-21T00:17:27,953 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:27,953 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/1853833cf2aa48349a736c8aec74142c is 50, key is test_row_0/B:col10/1732148245510/Put/seqid=0 2024-11-21T00:17:27,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742322_1498 (size=12949) 2024-11-21T00:17:27,965 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/1853833cf2aa48349a736c8aec74142c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1853833cf2aa48349a736c8aec74142c 2024-11-21T00:17:27,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:17:27,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:27,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:27,971 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into 1853833cf2aa48349a736c8aec74142c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:27,971 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:27,971 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148247945; duration=0sec 2024-11-21T00:17:27,971 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:27,971 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/060b0511fd4840818adef3a6071c936f is 50, key is test_row_0/A:col10/1732148246809/Put/seqid=0 2024-11-21T00:17:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742323_1499 (size=12301) 2024-11-21T00:17:28,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148308053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148308053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148308053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148308055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148308177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148308177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148308177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148308177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148308397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148308397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,406 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/060b0511fd4840818adef3a6071c936f 2024-11-21T00:17:28,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148308397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148308397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/7dc4a1df6cf846499df0a7a2ae63274c is 50, key is test_row_0/B:col10/1732148246809/Put/seqid=0 2024-11-21T00:17:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742324_1500 (size=12301) 2024-11-21T00:17:28,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148308716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148308716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148308716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148308716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:28,858 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/7dc4a1df6cf846499df0a7a2ae63274c 2024-11-21T00:17:28,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/8e6f77fae4b5448da67e62fca5d7622c is 50, key is test_row_0/C:col10/1732148246809/Put/seqid=0 2024-11-21T00:17:28,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742325_1501 (size=12301) 2024-11-21T00:17:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148309226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:29,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148309236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148309236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:29,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148309237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:29,273 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/8e6f77fae4b5448da67e62fca5d7622c 2024-11-21T00:17:29,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/060b0511fd4840818adef3a6071c936f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f 2024-11-21T00:17:29,291 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-21T00:17:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/7dc4a1df6cf846499df0a7a2ae63274c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c 2024-11-21T00:17:29,302 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-21T00:17:29,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/8e6f77fae4b5448da67e62fca5d7622c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c 2024-11-21T00:17:29,315 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-21T00:17:29,316 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6f23b76c1c42e5c3a6984684bda2f10b in 1345ms, sequenceid=294, compaction requested=true 2024-11-21T00:17:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T00:17:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T00:17:29,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T00:17:29,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4990 sec 2024-11-21T00:17:29,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.5150 sec 2024-11-21T00:17:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:29,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:29,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:29,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/61f9e2c5e5094dcd9089900ade0a370e is 50, key is test_row_0/A:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742326_1502 (size=14741) 2024-11-21T00:17:29,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/61f9e2c5e5094dcd9089900ade0a370e 2024-11-21T00:17:29,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/79a9c163828146598689035dcdfd0d86 is 50, key is test_row_0/B:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:29,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742327_1503 (size=12301) 2024-11-21T00:17:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148309933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:17:30,001 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:30,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T00:17:30,006 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:30,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:30,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:30,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148310059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:30,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/79a9c163828146598689035dcdfd0d86 2024-11-21T00:17:30,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aed7a92e8cc34949a645355b6bfe5786 is 50, key is test_row_0/C:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:30,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:17:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742328_1504 (size=12301) 2024-11-21T00:17:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148310257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148310261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148310261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,266 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,268 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,269 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148310267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,279 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148310286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:17:30,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:30,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:17:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aed7a92e8cc34949a645355b6bfe5786 2024-11-21T00:17:30,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148310608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:30,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/61f9e2c5e5094dcd9089900ade0a370e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e 2024-11-21T00:17:30,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e, entries=200, sequenceid=308, filesize=14.4 K 2024-11-21T00:17:30,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/79a9c163828146598689035dcdfd0d86 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86 2024-11-21T00:17:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:30,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T00:17:30,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/aed7a92e8cc34949a645355b6bfe5786 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786 2024-11-21T00:17:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:30,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T00:17:30,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6f23b76c1c42e5c3a6984684bda2f10b in 1035ms, sequenceid=308, compaction requested=true 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:30,689 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:30,689 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:30,700 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:30,700 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:30,701 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,701 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1853833cf2aa48349a736c8aec74142c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.7 K 2024-11-21T00:17:30,701 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:30,701 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:30,701 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,701 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5633aa96578546b9bdbf805dbe923b2e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=53.3 K 2024-11-21T00:17:30,701 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1853833cf2aa48349a736c8aec74142c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732148245481 2024-11-21T00:17:30,702 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5633aa96578546b9bdbf805dbe923b2e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732148244164 2024-11-21T00:17:30,702 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dc4a1df6cf846499df0a7a2ae63274c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732148246788 2024-11-21T00:17:30,702 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7242b6f8cb63469b945f3dc07e01275b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732148245481 2024-11-21T00:17:30,702 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a9c163828146598689035dcdfd0d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:30,702 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 060b0511fd4840818adef3a6071c936f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732148246788 2024-11-21T00:17:30,703 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f9e2c5e5094dcd9089900ade0a370e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:30,715 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:30,715 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f69c5661328e49e48e7b7ad8a324f5b4 is 50, key is test_row_0/B:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:30,735 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:30,735 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/b3988bc5988945b19d623f52b03dff14 is 50, key is test_row_0/A:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:30,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742329_1505 (size=13051) 2024-11-21T00:17:30,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742330_1506 (size=13051) 2024-11-21T00:17:30,803 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/b3988bc5988945b19d623f52b03dff14 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/b3988bc5988945b19d623f52b03dff14 2024-11-21T00:17:30,811 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into b3988bc5988945b19d623f52b03dff14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:30,812 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:30,812 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=12, startTime=1732148250689; duration=0sec 2024-11-21T00:17:30,812 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:30,812 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:30,812 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:30,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:17:30,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,813 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:30,815 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:30,815 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:30,815 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:30,815 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f1de50dcd20c4127a428faf51c686b12, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=48.5 K 2024-11-21T00:17:30,816 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1de50dcd20c4127a428faf51c686b12, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732148244164 2024-11-21T00:17:30,816 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f40c2e58814b779db1dd4f8eadec95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732148245481 2024-11-21T00:17:30,817 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6f77fae4b5448da67e62fca5d7622c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732148246788 2024-11-21T00:17:30,817 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting aed7a92e8cc34949a645355b6bfe5786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:30,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/438164a8005c4dc88fa236484244efd0 is 50, key is test_row_0/A:col10/1732148249851/Put/seqid=0 2024-11-21T00:17:30,842 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:30,843 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/0595c243b80449568f64bfda8d8fbfdc is 50, key is test_row_0/C:col10/1732148248003/Put/seqid=0 2024-11-21T00:17:30,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742331_1507 (size=12301) 2024-11-21T00:17:30,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742332_1508 (size=13051) 2024-11-21T00:17:30,884 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/0595c243b80449568f64bfda8d8fbfdc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/0595c243b80449568f64bfda8d8fbfdc 2024-11-21T00:17:30,892 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into 0595c243b80449568f64bfda8d8fbfdc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:30,892 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:30,892 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=12, startTime=1732148250689; duration=0sec 2024-11-21T00:17:30,892 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:30,892 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:31,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:31,207 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f69c5661328e49e48e7b7ad8a324f5b4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f69c5661328e49e48e7b7ad8a324f5b4 2024-11-21T00:17:31,234 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into f69c5661328e49e48e7b7ad8a324f5b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:31,235 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:31,235 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148250689; duration=0sec 2024-11-21T00:17:31,235 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:31,235 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:31,273 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/438164a8005c4dc88fa236484244efd0 2024-11-21T00:17:31,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148311267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4237143464ab4bfd980389b2389b08bb is 50, key is test_row_0/B:col10/1732148249851/Put/seqid=0 2024-11-21T00:17:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742333_1509 (size=12301) 2024-11-21T00:17:31,336 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4237143464ab4bfd980389b2389b08bb 2024-11-21T00:17:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/139b960f5c2c409aba831ee930df63a7 is 50, key is test_row_0/C:col10/1732148249851/Put/seqid=0 2024-11-21T00:17:31,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742334_1510 (size=12301) 2024-11-21T00:17:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148311386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:31,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148311597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:31,761 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/139b960f5c2c409aba831ee930df63a7 2024-11-21T00:17:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/438164a8005c4dc88fa236484244efd0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0 2024-11-21T00:17:31,777 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T00:17:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/4237143464ab4bfd980389b2389b08bb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb 2024-11-21T00:17:31,780 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T00:17:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/139b960f5c2c409aba831ee930df63a7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7 2024-11-21T00:17:31,788 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T00:17:31,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6f23b76c1c42e5c3a6984684bda2f10b in 985ms, sequenceid=328, compaction requested=false 2024-11-21T00:17:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T00:17:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T00:17:31,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T00:17:31,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7950 sec 2024-11-21T00:17:31,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8030 sec 2024-11-21T00:17:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:31,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:31,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:31,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/793a2f722e164e169d3f8d83a26e5fd2 is 50, key is test_row_0/A:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:31,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742335_1511 (size=14741) 2024-11-21T00:17:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148312117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:17:32,162 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:32,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T00:17:32,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:32,174 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:32,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148312229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52706 deadline: 1732148312278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,290 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52730 deadline: 1732148312280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1732148312292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,297 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,300 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1732148312298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,316 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in 6f23b76c1c42e5c3a6984684bda2f10b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:32,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:32,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/793a2f722e164e169d3f8d83a26e5fd2 2024-11-21T00:17:32,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e73521249c184d5982a7a5cafa1c112c is 50, key is test_row_0/B:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148312447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742336_1512 (size=12301) 2024-11-21T00:17:32,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148312767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:32,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:32,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:32,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e73521249c184d5982a7a5cafa1c112c 2024-11-21T00:17:32,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/20e7b7ef2c81421ca86b3e5e12b95ec2 is 50, key is test_row_0/C:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:32,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742337_1513 (size=12301) 2024-11-21T00:17:32,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:32,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:33,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148313281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:33,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:33,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/20e7b7ef2c81421ca86b3e5e12b95ec2 2024-11-21T00:17:33,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/793a2f722e164e169d3f8d83a26e5fd2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2 2024-11-21T00:17:33,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2, entries=200, sequenceid=344, filesize=14.4 K 2024-11-21T00:17:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/e73521249c184d5982a7a5cafa1c112c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c 2024-11-21T00:17:33,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T00:17:33,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/20e7b7ef2c81421ca86b3e5e12b95ec2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2 2024-11-21T00:17:33,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T00:17:33,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6f23b76c1c42e5c3a6984684bda2f10b in 1436ms, sequenceid=344, compaction requested=true 2024-11-21T00:17:33,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:33,359 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:33,360 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:33,360 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:33,360 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,361 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/b3988bc5988945b19d623f52b03dff14, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=39.2 K 2024-11-21T00:17:33,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3988bc5988945b19d623f52b03dff14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:33,361 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:33,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438164a8005c4dc88fa236484244efd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148249851 2024-11-21T00:17:33,362 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 793a2f722e164e169d3f8d83a26e5fd2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251234 2024-11-21T00:17:33,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:33,362 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:33,362 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:33,362 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,362 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f69c5661328e49e48e7b7ad8a324f5b4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.8 K 2024-11-21T00:17:33,363 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f69c5661328e49e48e7b7ad8a324f5b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:33,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:33,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:33,363 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4237143464ab4bfd980389b2389b08bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148249851 2024-11-21T00:17:33,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:33,363 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e73521249c184d5982a7a5cafa1c112c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251249 2024-11-21T00:17:33,373 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:33,374 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/1596f0679955449f97f40a09f7f110dc is 50, key is test_row_0/A:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:33,383 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:33,384 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/ed3a4c106c4b4d0d9e7080966e723319 is 50, key is test_row_0/B:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742338_1514 (size=13153) 2024-11-21T00:17:33,416 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/1596f0679955449f97f40a09f7f110dc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1596f0679955449f97f40a09f7f110dc 2024-11-21T00:17:33,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742339_1515 (size=13153) 2024-11-21T00:17:33,448 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 1596f0679955449f97f40a09f7f110dc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:33,448 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:33,448 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148253359; duration=0sec 2024-11-21T00:17:33,448 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:33,449 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:33,449 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:33,453 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:33,453 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:33,453 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,453 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/0595c243b80449568f64bfda8d8fbfdc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.8 K 2024-11-21T00:17:33,453 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0595c243b80449568f64bfda8d8fbfdc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732148248003 2024-11-21T00:17:33,454 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139b960f5c2c409aba831ee930df63a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148249851 2024-11-21T00:17:33,454 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e7b7ef2c81421ca86b3e5e12b95ec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251249 2024-11-21T00:17:33,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:17:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:33,469 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:33,475 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:33,475 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/a9b09f483d3f4ba88ab9c06533f6bc41 is 50, key is test_row_0/C:col10/1732148251922/Put/seqid=0 2024-11-21T00:17:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d939a6cd42764266a7220f4ce32a96a7 is 50, key is test_row_0/A:col10/1732148252096/Put/seqid=0 2024-11-21T00:17:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742340_1516 (size=13153) 2024-11-21T00:17:33,557 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/a9b09f483d3f4ba88ab9c06533f6bc41 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/a9b09f483d3f4ba88ab9c06533f6bc41 2024-11-21T00:17:33,574 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into a9b09f483d3f4ba88ab9c06533f6bc41(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:33,574 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:33,574 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148253363; duration=0sec 2024-11-21T00:17:33,574 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:33,574 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:33,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742341_1517 (size=12301) 2024-11-21T00:17:33,598 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d939a6cd42764266a7220f4ce32a96a7 2024-11-21T00:17:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/35cc5b65d908406a8f80810f64696867 is 50, key is test_row_0/B:col10/1732148252096/Put/seqid=0 2024-11-21T00:17:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742342_1518 (size=12301) 2024-11-21T00:17:33,679 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/35cc5b65d908406a8f80810f64696867 2024-11-21T00:17:33,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b88568665edc4ca4a59bb3d4635b832f is 50, key is test_row_0/C:col10/1732148252096/Put/seqid=0 2024-11-21T00:17:33,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742343_1519 (size=12301) 2024-11-21T00:17:33,869 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/ed3a4c106c4b4d0d9e7080966e723319 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/ed3a4c106c4b4d0d9e7080966e723319 2024-11-21T00:17:33,904 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into ed3a4c106c4b4d0d9e7080966e723319(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:33,904 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:33,904 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148253361; duration=0sec 2024-11-21T00:17:33,904 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:33,904 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:34,172 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b88568665edc4ca4a59bb3d4635b832f 2024-11-21T00:17:34,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d939a6cd42764266a7220f4ce32a96a7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7 2024-11-21T00:17:34,216 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T00:17:34,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/35cc5b65d908406a8f80810f64696867 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867 2024-11-21T00:17:34,237 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T00:17:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b88568665edc4ca4a59bb3d4635b832f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f 2024-11-21T00:17:34,242 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T00:17:34,243 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 6f23b76c1c42e5c3a6984684bda2f10b in 775ms, sequenceid=363, compaction requested=false 2024-11-21T00:17:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T00:17:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T00:17:34,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T00:17:34,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-11-21T00:17:34,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0880 sec 2024-11-21T00:17:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:17:34,326 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:34,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T00:17:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:34,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:34,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:34,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:34,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:34,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:34,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0b9a0094532f45aaa8fb23e2f918bed3 is 50, key is test_row_0/A:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742344_1520 (size=12301) 2024-11-21T00:17:34,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0b9a0094532f45aaa8fb23e2f918bed3 2024-11-21T00:17:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:34,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/22031916f55f4c92a3fad15d5bb3b931 is 50, key is test_row_0/B:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742345_1521 (size=12301) 2024-11-21T00:17:34,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/22031916f55f4c92a3fad15d5bb3b931 2024-11-21T00:17:34,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:34,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:34,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/5a88d8195012414b9963fc8ad31728bf is 50, key is test_row_0/C:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742346_1522 (size=12301) 2024-11-21T00:17:34,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/5a88d8195012414b9963fc8ad31728bf 2024-11-21T00:17:34,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/0b9a0094532f45aaa8fb23e2f918bed3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3 2024-11-21T00:17:34,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T00:17:34,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/22031916f55f4c92a3fad15d5bb3b931 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931 2024-11-21T00:17:34,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T00:17:34,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/5a88d8195012414b9963fc8ad31728bf as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf 2024-11-21T00:17:34,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T00:17:34,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for 6f23b76c1c42e5c3a6984684bda2f10b in 226ms, sequenceid=375, compaction requested=true 2024-11-21T00:17:34,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:34,591 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:34,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:34,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:34,591 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:34,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:34,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:34,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:34,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:34,596 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:34,596 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/B is initiating minor compaction (all files) 2024-11-21T00:17:34,596 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/B in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,596 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/ed3a4c106c4b4d0d9e7080966e723319, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.9 K 2024-11-21T00:17:34,598 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:34,598 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/A is initiating minor compaction (all files) 2024-11-21T00:17:34,598 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/A in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,598 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3a4c106c4b4d0d9e7080966e723319, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251249 2024-11-21T00:17:34,599 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1596f0679955449f97f40a09f7f110dc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.9 K 2024-11-21T00:17:34,599 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 35cc5b65d908406a8f80810f64696867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732148252078 2024-11-21T00:17:34,599 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1596f0679955449f97f40a09f7f110dc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251249 2024-11-21T00:17:34,599 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting d939a6cd42764266a7220f4ce32a96a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732148252078 2024-11-21T00:17:34,599 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 22031916f55f4c92a3fad15d5bb3b931, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732148254341 2024-11-21T00:17:34,600 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9a0094532f45aaa8fb23e2f918bed3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732148254341 2024-11-21T00:17:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:34,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:34,616 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:34,617 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/005900095d104302938cfb907c793774 is 50, key is test_row_0/A:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,622 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:34,623 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f8a14bcd269b4be393117a9e684c9e2f is 50, key is test_row_0/B:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:34,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:34,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:34,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742347_1523 (size=13255) 2024-11-21T00:17:34,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742348_1524 (size=13255) 2024-11-21T00:17:34,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d0ba87cc29124d07909deffcaa6b478d is 50, key is test_row_0/A:col10/1732148254551/Put/seqid=0 2024-11-21T00:17:34,700 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/005900095d104302938cfb907c793774 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/005900095d104302938cfb907c793774 2024-11-21T00:17:34,708 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/A of 6f23b76c1c42e5c3a6984684bda2f10b into 005900095d104302938cfb907c793774(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:34,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:34,708 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/A, priority=13, startTime=1732148254590; duration=0sec 2024-11-21T00:17:34,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:34,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:34,708 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:34,711 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:34,711 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): 6f23b76c1c42e5c3a6984684bda2f10b/C is initiating minor compaction (all files) 2024-11-21T00:17:34,711 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f23b76c1c42e5c3a6984684bda2f10b/C in TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,711 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/a9b09f483d3f4ba88ab9c06533f6bc41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp, totalSize=36.9 K 2024-11-21T00:17:34,711 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b09f483d3f4ba88ab9c06533f6bc41, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732148251249 2024-11-21T00:17:34,712 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b88568665edc4ca4a59bb3d4635b832f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732148252078 2024-11-21T00:17:34,712 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a88d8195012414b9963fc8ad31728bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732148254341 2024-11-21T00:17:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742349_1525 (size=14741) 2024-11-21T00:17:34,739 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f23b76c1c42e5c3a6984684bda2f10b#C#compaction#447 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:34,739 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/d58d68b2b3c04da19c0c48abdba09cd3 is 50, key is test_row_0/C:col10/1732148254352/Put/seqid=0 2024-11-21T00:17:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742350_1526 (size=13255) 2024-11-21T00:17:34,784 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/d58d68b2b3c04da19c0c48abdba09cd3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/d58d68b2b3c04da19c0c48abdba09cd3 2024-11-21T00:17:34,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,801 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/C of 6f23b76c1c42e5c3a6984684bda2f10b into d58d68b2b3c04da19c0c48abdba09cd3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:34,801 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:34,801 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/C, priority=13, startTime=1732148254591; duration=0sec 2024-11-21T00:17:34,801 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:34,801 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:34,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148314869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:34,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:34,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:34,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:35,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148314997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:35,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:35,078 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f8a14bcd269b4be393117a9e684c9e2f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f8a14bcd269b4be393117a9e684c9e2f 2024-11-21T00:17:35,092 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f23b76c1c42e5c3a6984684bda2f10b/B of 6f23b76c1c42e5c3a6984684bda2f10b into f8a14bcd269b4be393117a9e684c9e2f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:35,092 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:35,092 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., storeName=6f23b76c1c42e5c3a6984684bda2f10b/B, priority=13, startTime=1732148254591; duration=0sec 2024-11-21T00:17:35,093 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:35,093 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:35,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:35,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:35,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:35,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:35,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:35,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:35,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d0ba87cc29124d07909deffcaa6b478d 2024-11-21T00:17:35,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f935edbc36e94b74af145f00a43c6a72 is 50, key is test_row_0/B:col10/1732148254551/Put/seqid=0 2024-11-21T00:17:35,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742351_1527 (size=12301) 2024-11-21T00:17:35,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f935edbc36e94b74af145f00a43c6a72 2024-11-21T00:17:35,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b337c8639161443582c2e66e4e057563 is 50, key is test_row_0/C:col10/1732148254551/Put/seqid=0 2024-11-21T00:17:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148315216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:35,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742352_1528 (size=12301) 2024-11-21T00:17:35,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b337c8639161443582c2e66e4e057563 2024-11-21T00:17:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/d0ba87cc29124d07909deffcaa6b478d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d0ba87cc29124d07909deffcaa6b478d 2024-11-21T00:17:35,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d0ba87cc29124d07909deffcaa6b478d, entries=200, sequenceid=393, filesize=14.4 K 2024-11-21T00:17:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/f935edbc36e94b74af145f00a43c6a72 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f935edbc36e94b74af145f00a43c6a72 2024-11-21T00:17:35,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f935edbc36e94b74af145f00a43c6a72, entries=150, sequenceid=393, filesize=12.0 K 2024-11-21T00:17:35,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/b337c8639161443582c2e66e4e057563 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b337c8639161443582c2e66e4e057563 2024-11-21T00:17:35,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b337c8639161443582c2e66e4e057563, entries=150, sequenceid=393, filesize=12.0 K 2024-11-21T00:17:35,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6f23b76c1c42e5c3a6984684bda2f10b in 632ms, sequenceid=393, compaction requested=false 2024-11-21T00:17:35,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:35,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:17:35,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:35,272 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:35,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/8a63e60009ae4b55bfc8bef71680dc18 is 50, key is test_row_0/A:col10/1732148254707/Put/seqid=0 2024-11-21T00:17:35,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742353_1529 (size=12301) 2024-11-21T00:17:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:35,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. as already flushing 2024-11-21T00:17:35,701 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/8a63e60009ae4b55bfc8bef71680dc18 2024-11-21T00:17:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/0b4662f8ed2e4531b73807afa79624c2 is 50, key is test_row_0/B:col10/1732148254707/Put/seqid=0 2024-11-21T00:17:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148315740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:35,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742354_1530 (size=12301) 2024-11-21T00:17:35,768 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/0b4662f8ed2e4531b73807afa79624c2 2024-11-21T00:17:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/45bafc148a7148879cc04b1a30e51285 is 50, key is test_row_0/C:col10/1732148254707/Put/seqid=0 2024-11-21T00:17:35,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742355_1531 (size=12301) 2024-11-21T00:17:35,834 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/45bafc148a7148879cc04b1a30e51285 2024-11-21T00:17:35,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/8a63e60009ae4b55bfc8bef71680dc18 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/8a63e60009ae4b55bfc8bef71680dc18 2024-11-21T00:17:35,847 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/8a63e60009ae4b55bfc8bef71680dc18, entries=150, sequenceid=410, filesize=12.0 K 2024-11-21T00:17:35,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/0b4662f8ed2e4531b73807afa79624c2 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/0b4662f8ed2e4531b73807afa79624c2 2024-11-21T00:17:35,858 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/0b4662f8ed2e4531b73807afa79624c2, entries=150, sequenceid=410, filesize=12.0 K 2024-11-21T00:17:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/45bafc148a7148879cc04b1a30e51285 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/45bafc148a7148879cc04b1a30e51285 2024-11-21T00:17:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52722 deadline: 1732148315856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:17:35,882 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/45bafc148a7148879cc04b1a30e51285, entries=150, sequenceid=410, filesize=12.0 K 2024-11-21T00:17:35,883 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6f23b76c1c42e5c3a6984684bda2f10b in 611ms, sequenceid=410, compaction requested=true 2024-11-21T00:17:35,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:35,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:35,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T00:17:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T00:17:35,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T00:17:35,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5550 sec 2024-11-21T00:17:35,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5620 sec 2024-11-21T00:17:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:36,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T00:17:36,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:36,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:36,080 INFO [Thread-1967 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1967. 2024-11-21T00:17:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:36,080 DEBUG [Thread-1967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:36,080 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,080 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,082 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-21T00:17:36,082 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,082 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,082 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,089 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-11-21T00:17:36,089 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,089 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,090 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,090 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-21T00:17:36,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,090 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,090 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,090 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/581e565856194b128a5fbde780b74739 is 50, key is test_row_0/A:col10/1732148256078/Put/seqid=0 2024-11-21T00:17:36,100 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-11-21T00:17:36,100 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,100 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,101 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,101 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-21T00:17:36,101 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,101 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,101 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742356_1532 (size=14741) 2024-11-21T00:17:36,318 INFO [Thread-1969 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1969. 2024-11-21T00:17:36,318 DEBUG [Thread-1969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,318 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,318 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,319 INFO [Thread-1971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1971. 2024-11-21T00:17:36,319 DEBUG [Thread-1971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,319 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,319 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,328 INFO [Thread-1965 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1965. 2024-11-21T00:17:36,328 DEBUG [Thread-1965 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,328 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,328 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,338 INFO [Thread-1973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1973. 2024-11-21T00:17:36,338 DEBUG [Thread-1973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:17:36,338 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,338 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:17:36,476 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 904 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2705 rows 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2683 rows 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 897 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2691 rows 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2682 rows 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 901 2024-11-21T00:17:36,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2703 rows 2024-11-21T00:17:36,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:17:36,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:17:36,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:36,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:17:36,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:36,488 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148256488"}]},"ts":"1732148256488"} 2024-11-21T00:17:36,491 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:17:36,491 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:17:36,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:17:36,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, UNASSIGN}] 2024-11-21T00:17:36,495 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, UNASSIGN 2024-11-21T00:17:36,496 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=6f23b76c1c42e5c3a6984684bda2f10b, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:36,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, UNASSIGN because future has completed 2024-11-21T00:17:36,498 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:17:36,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:36,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/581e565856194b128a5fbde780b74739 2024-11-21T00:17:36,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a05663ed563148e1b9ed019ac974f85b is 50, key is test_row_0/B:col10/1732148256078/Put/seqid=0 2024-11-21T00:17:36,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742357_1533 (size=12301) 2024-11-21T00:17:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:36,651 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:36,651 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:17:36,651 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 6f23b76c1c42e5c3a6984684bda2f10b, disabling compactions & flushes 2024-11-21T00:17:36,651 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:36,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a05663ed563148e1b9ed019ac974f85b 2024-11-21T00:17:36,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f8e2051dfa1f44398db388ddb4256ae7 is 50, key is test_row_0/C:col10/1732148256078/Put/seqid=0 2024-11-21T00:17:36,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742358_1534 (size=12301) 2024-11-21T00:17:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:37,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f8e2051dfa1f44398db388ddb4256ae7 2024-11-21T00:17:37,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/581e565856194b128a5fbde780b74739 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/581e565856194b128a5fbde780b74739 2024-11-21T00:17:37,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/581e565856194b128a5fbde780b74739, entries=200, sequenceid=429, filesize=14.4 K 2024-11-21T00:17:37,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/a05663ed563148e1b9ed019ac974f85b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a05663ed563148e1b9ed019ac974f85b 2024-11-21T00:17:37,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a05663ed563148e1b9ed019ac974f85b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T00:17:37,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/f8e2051dfa1f44398db388ddb4256ae7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f8e2051dfa1f44398db388ddb4256ae7 2024-11-21T00:17:37,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f8e2051dfa1f44398db388ddb4256ae7, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T00:17:37,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 6f23b76c1c42e5c3a6984684bda2f10b in 1281ms, sequenceid=429, compaction requested=true 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f23b76c1c42e5c3a6984684bda2f10b: 2024-11-21T00:17:37,361 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:A 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. after waiting 0 ms 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:B, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:B 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. because compaction request was cancelled 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f23b76c1c42e5c3a6984684bda2f10b:C, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:37,361 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f23b76c1c42e5c3a6984684bda2f10b:C 2024-11-21T00:17:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:37,361 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 6f23b76c1c42e5c3a6984684bda2f10b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=A 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=B 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f23b76c1c42e5c3a6984684bda2f10b, store=C 2024-11-21T00:17:37,361 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:37,367 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cfae7a080a894fa29c20eb43a86c028d is 50, key is test_row_0/A:col10/1732148256326/Put/seqid=0 2024-11-21T00:17:37,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742359_1535 (size=9857) 2024-11-21T00:17:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:37,778 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cfae7a080a894fa29c20eb43a86c028d 2024-11-21T00:17:37,783 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/24bbe9c7b9c346f7b9f8159685ba7d52 is 50, key is test_row_0/B:col10/1732148256326/Put/seqid=0 2024-11-21T00:17:37,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742360_1536 (size=9857) 2024-11-21T00:17:38,187 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/24bbe9c7b9c346f7b9f8159685ba7d52 2024-11-21T00:17:38,192 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/430195a4f85c4fbe995e1daf449c4d9a is 50, key is test_row_0/C:col10/1732148256326/Put/seqid=0 2024-11-21T00:17:38,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742361_1537 (size=9857) 2024-11-21T00:17:38,595 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/430195a4f85c4fbe995e1daf449c4d9a 2024-11-21T00:17:38,598 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/A/cfae7a080a894fa29c20eb43a86c028d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cfae7a080a894fa29c20eb43a86c028d 2024-11-21T00:17:38,602 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cfae7a080a894fa29c20eb43a86c028d, entries=100, sequenceid=436, filesize=9.6 K 2024-11-21T00:17:38,603 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/B/24bbe9c7b9c346f7b9f8159685ba7d52 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/24bbe9c7b9c346f7b9f8159685ba7d52 2024-11-21T00:17:38,606 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/24bbe9c7b9c346f7b9f8159685ba7d52, entries=100, sequenceid=436, filesize=9.6 K 2024-11-21T00:17:38,606 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/.tmp/C/430195a4f85c4fbe995e1daf449c4d9a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/430195a4f85c4fbe995e1daf449c4d9a 2024-11-21T00:17:38,610 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/430195a4f85c4fbe995e1daf449c4d9a, entries=100, sequenceid=436, filesize=9.6 K 2024-11-21T00:17:38,611 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6f23b76c1c42e5c3a6984684bda2f10b in 1249ms, sequenceid=436, compaction requested=true 2024-11-21T00:17:38,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a2b26bbdc2d24634aaa10186f511e4a8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0fbea2e9cf664845b43a318fb8080db7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5614126d4cfb49fb9d205fd4bf1fea71, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1e9798f1b596467ca11f1ccb51e8e78a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/de85bcd286ad47f5943fa595e9a9bf74, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5633aa96578546b9bdbf805dbe923b2e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/b3988bc5988945b19d623f52b03dff14, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1596f0679955449f97f40a09f7f110dc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3] to archive 2024-11-21T00:17:38,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:38,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a3a3934ba0e643ec9deef117e9ad4ea9 2024-11-21T00:17:38,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d410ba20a01443ba9d95c3734982eecf 2024-11-21T00:17:38,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a2b26bbdc2d24634aaa10186f511e4a8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a2b26bbdc2d24634aaa10186f511e4a8 2024-11-21T00:17:38,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0ccdcb4f16554365ae37e1c835e3a1ad 2024-11-21T00:17:38,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bc8a6ec9b27b45b99b678e3afbbfef25 2024-11-21T00:17:38,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/176e7a99f1a44b1187052373f4d197db 2024-11-21T00:17:38,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0fbea2e9cf664845b43a318fb8080db7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0fbea2e9cf664845b43a318fb8080db7 2024-11-21T00:17:38,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bd37349bd4d64c5882e68de9f7ef125a 2024-11-21T00:17:38,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cc5855c1db8543308bbbc33a386ae65c 2024-11-21T00:17:38,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5614126d4cfb49fb9d205fd4bf1fea71 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5614126d4cfb49fb9d205fd4bf1fea71 2024-11-21T00:17:38,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a09a048f9c35403196a0e702e12eda96 2024-11-21T00:17:38,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a24dd4e2172543068a80a7afe5ab4bc3 2024-11-21T00:17:38,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1e9798f1b596467ca11f1ccb51e8e78a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1e9798f1b596467ca11f1ccb51e8e78a 2024-11-21T00:17:38,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/546c1a32274548088985e60d8968fa30 2024-11-21T00:17:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:38,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/a388895ff8ab43d5a9ab4ccc7406c1a2 2024-11-21T00:17:38,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/de85bcd286ad47f5943fa595e9a9bf74 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/de85bcd286ad47f5943fa595e9a9bf74 2024-11-21T00:17:38,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/56286baf8c0941c8b28b17c59db88e8b 2024-11-21T00:17:38,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5633aa96578546b9bdbf805dbe923b2e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/5633aa96578546b9bdbf805dbe923b2e 2024-11-21T00:17:38,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/bb65922c969f42839c4aba7bc2fa58e6 2024-11-21T00:17:38,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/7242b6f8cb63469b945f3dc07e01275b 2024-11-21T00:17:38,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/060b0511fd4840818adef3a6071c936f 2024-11-21T00:17:38,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/61f9e2c5e5094dcd9089900ade0a370e 2024-11-21T00:17:38,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/b3988bc5988945b19d623f52b03dff14 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/b3988bc5988945b19d623f52b03dff14 2024-11-21T00:17:38,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/438164a8005c4dc88fa236484244efd0 2024-11-21T00:17:38,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/793a2f722e164e169d3f8d83a26e5fd2 2024-11-21T00:17:38,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1596f0679955449f97f40a09f7f110dc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/1596f0679955449f97f40a09f7f110dc 2024-11-21T00:17:38,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d939a6cd42764266a7220f4ce32a96a7 2024-11-21T00:17:38,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/0b9a0094532f45aaa8fb23e2f918bed3 2024-11-21T00:17:38,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4f726a266b7b4bc5ba61c9efd17cb3d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/48f2cde2768e4dec870fd245665c877d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/d49f50e1bfb641b99036adbf7d4b722b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9c35ba9d82ff41f5807f2b7d1cf1070f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/52d92cd308cf4fbc851e9b7845cfe62f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1853833cf2aa48349a736c8aec74142c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f69c5661328e49e48e7b7ad8a324f5b4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/ed3a4c106c4b4d0d9e7080966e723319, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931] to archive 2024-11-21T00:17:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:38,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/08c8281337e8418db5ae2bca7d04229d 2024-11-21T00:17:38,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/c7be262cd9004e93acb15e3c3a9639ba 2024-11-21T00:17:38,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4f726a266b7b4bc5ba61c9efd17cb3d3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4f726a266b7b4bc5ba61c9efd17cb3d3 2024-11-21T00:17:38,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9b86de7fac9243b9ba3dcb87a0a8640f 2024-11-21T00:17:38,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e6c53d585061489490478e6e96531dab 2024-11-21T00:17:38,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/48f2cde2768e4dec870fd245665c877d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/48f2cde2768e4dec870fd245665c877d 2024-11-21T00:17:38,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/8ae66b8f164a43558d14e9f803a79d49 2024-11-21T00:17:38,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5bb8557b8bfc4d758a6c68a034a5dd65 2024-11-21T00:17:38,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/d49f50e1bfb641b99036adbf7d4b722b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/d49f50e1bfb641b99036adbf7d4b722b 2024-11-21T00:17:38,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9ff929712ea94e328d3c8d91df6f7f28 2024-11-21T00:17:38,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/74495f56497e4a99bd19da6c104b9e2c 2024-11-21T00:17:38,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9c35ba9d82ff41f5807f2b7d1cf1070f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/9c35ba9d82ff41f5807f2b7d1cf1070f 2024-11-21T00:17:38,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a446f40ae3884d6ca1bc988c135abc9a 2024-11-21T00:17:38,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/b05f1c9741164496aa70071426dfa357 2024-11-21T00:17:38,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1a89edb77f084764bfde01f254435dbc 2024-11-21T00:17:38,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/52d92cd308cf4fbc851e9b7845cfe62f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/52d92cd308cf4fbc851e9b7845cfe62f 2024-11-21T00:17:38,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/5f671a12bc144567baf69b3779a0d05e 2024-11-21T00:17:38,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/26ffa45153c94dceb0861981f452ba7b 2024-11-21T00:17:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1853833cf2aa48349a736c8aec74142c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/1853833cf2aa48349a736c8aec74142c 2024-11-21T00:17:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4c0fade723714e099ed33d6376f2cbbc 2024-11-21T00:17:38,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/7dc4a1df6cf846499df0a7a2ae63274c 2024-11-21T00:17:38,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f69c5661328e49e48e7b7ad8a324f5b4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f69c5661328e49e48e7b7ad8a324f5b4 2024-11-21T00:17:38,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/79a9c163828146598689035dcdfd0d86 2024-11-21T00:17:38,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/4237143464ab4bfd980389b2389b08bb 2024-11-21T00:17:38,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/ed3a4c106c4b4d0d9e7080966e723319 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/ed3a4c106c4b4d0d9e7080966e723319 2024-11-21T00:17:38,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/e73521249c184d5982a7a5cafa1c112c 2024-11-21T00:17:38,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/35cc5b65d908406a8f80810f64696867 2024-11-21T00:17:38,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/22031916f55f4c92a3fad15d5bb3b931 2024-11-21T00:17:38,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/babaae6ba1ac4a919723b48f8e66bb97, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/dd89c2b31c644935b0ca612e5cbd779b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/710eaf4896c844ce9effbb411179fea4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/bb120ee228134dc781b3a0942f916a92, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cf0afeb8c5e9418c8b5d4006b56d91a2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f1de50dcd20c4127a428faf51c686b12, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/0595c243b80449568f64bfda8d8fbfdc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/a9b09f483d3f4ba88ab9c06533f6bc41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf] to archive 2024-11-21T00:17:38,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:17:38,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/ff2b5e9635d94fcea54c4700e4ba880c 2024-11-21T00:17:38,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/46385c0e4363479390bbf06de8916d00 2024-11-21T00:17:38,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/babaae6ba1ac4a919723b48f8e66bb97 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/babaae6ba1ac4a919723b48f8e66bb97 2024-11-21T00:17:38,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1564610af7b54390851c95ba8c75a36e 2024-11-21T00:17:38,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cef1079c676f4291932cb9c6dde07c2d 2024-11-21T00:17:38,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/dd89c2b31c644935b0ca612e5cbd779b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/dd89c2b31c644935b0ca612e5cbd779b 2024-11-21T00:17:38,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/61d1d8b3a55244299506a162867074a9 2024-11-21T00:17:38,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/197726ed3fcd48289240d838ae530d81 2024-11-21T00:17:38,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/710eaf4896c844ce9effbb411179fea4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/710eaf4896c844ce9effbb411179fea4 2024-11-21T00:17:38,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/71ee95019a094a25b5f75c91a63894af 2024-11-21T00:17:38,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aabd6f36c7f24ad486ac50558fe50827 2024-11-21T00:17:38,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/bb120ee228134dc781b3a0942f916a92 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/bb120ee228134dc781b3a0942f916a92 2024-11-21T00:17:38,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/9a4d948bce114454bea3b728f7369518 2024-11-21T00:17:38,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/1095e0ef5475489cb86b6b05d6a6c388 2024-11-21T00:17:38,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cf0afeb8c5e9418c8b5d4006b56d91a2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/cf0afeb8c5e9418c8b5d4006b56d91a2 2024-11-21T00:17:38,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f48e65b95bc944fdacc8dc0b361db225 2024-11-21T00:17:38,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/e5d2bdcebfa146d4ba5bc55e7db0059f 2024-11-21T00:17:38,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f1de50dcd20c4127a428faf51c686b12 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f1de50dcd20c4127a428faf51c686b12 2024-11-21T00:17:38,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/114197629c574859b420d4b6f1dd7cac 2024-11-21T00:17:38,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/67f40c2e58814b779db1dd4f8eadec95 2024-11-21T00:17:38,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/8e6f77fae4b5448da67e62fca5d7622c 2024-11-21T00:17:38,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/0595c243b80449568f64bfda8d8fbfdc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/0595c243b80449568f64bfda8d8fbfdc 2024-11-21T00:17:38,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/aed7a92e8cc34949a645355b6bfe5786 2024-11-21T00:17:38,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/139b960f5c2c409aba831ee930df63a7 2024-11-21T00:17:38,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/a9b09f483d3f4ba88ab9c06533f6bc41 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/a9b09f483d3f4ba88ab9c06533f6bc41 2024-11-21T00:17:38,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/20e7b7ef2c81421ca86b3e5e12b95ec2 2024-11-21T00:17:38,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b88568665edc4ca4a59bb3d4635b832f 2024-11-21T00:17:38,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/5a88d8195012414b9963fc8ad31728bf 2024-11-21T00:17:38,729 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/recovered.edits/439.seqid, newMaxSeqId=439, maxSeqId=1 2024-11-21T00:17:38,729 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. 2024-11-21T00:17:38,730 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 6f23b76c1c42e5c3a6984684bda2f10b: Waiting for close lock at 1732148256651Running coprocessor pre-close hooks at 1732148256651Disabling compacts and flushes for region at 1732148256651Disabling writes for close at 1732148257361 (+710 ms)Obtaining lock to block concurrent updates at 1732148257361Preparing flush snapshotting stores in 6f23b76c1c42e5c3a6984684bda2f10b at 1732148257361Finished memstore snapshotting TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732148257361Flushing stores of TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b. at 1732148257362 (+1 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/A: creating writer at 1732148257362Flushing 6f23b76c1c42e5c3a6984684bda2f10b/A: appending metadata at 1732148257366 (+4 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/A: closing flushed file at 1732148257366Flushing 6f23b76c1c42e5c3a6984684bda2f10b/B: creating writer at 1732148257781 (+415 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/B: appending metadata at 1732148257783 (+2 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/B: closing flushed file at 1732148257783Flushing 6f23b76c1c42e5c3a6984684bda2f10b/C: creating writer at 1732148258189 (+406 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/C: appending metadata at 1732148258191 (+2 ms)Flushing 6f23b76c1c42e5c3a6984684bda2f10b/C: closing flushed file at 1732148258191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26d87ec4: reopening flushed file at 1732148258598 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cc8b1e9: reopening flushed file at 1732148258602 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@110ca07b: reopening flushed file at 1732148258606 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6f23b76c1c42e5c3a6984684bda2f10b in 1249ms, sequenceid=436, compaction requested=true at 1732148258611 (+5 ms)Writing region close event to WAL at 1732148258723 (+112 ms)Running coprocessor post-close hooks at 1732148258729 (+6 ms)Closed at 1732148258729 2024-11-21T00:17:38,733 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:38,734 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=6f23b76c1c42e5c3a6984684bda2f10b, regionState=CLOSED 2024-11-21T00:17:38,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T00:17:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 6f23b76c1c42e5c3a6984684bda2f10b, server=5ed4808ef0e6,40705,1732148113864 in 2.2390 sec 2024-11-21T00:17:38,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-21T00:17:38,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f23b76c1c42e5c3a6984684bda2f10b, UNASSIGN in 2.2460 sec 2024-11-21T00:17:38,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T00:17:38,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2500 sec 2024-11-21T00:17:38,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148258746"}]},"ts":"1732148258746"} 2024-11-21T00:17:38,749 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:17:38,758 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:17:38,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2810 sec 2024-11-21T00:17:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:17:40,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:40,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:17:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T00:17:40,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:40,682 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/recovered.edits] 2024-11-21T00:17:40,684 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/005900095d104302938cfb907c793774 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/005900095d104302938cfb907c793774 2024-11-21T00:17:40,685 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/581e565856194b128a5fbde780b74739 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/581e565856194b128a5fbde780b74739 2024-11-21T00:17:40,686 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/8a63e60009ae4b55bfc8bef71680dc18 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/8a63e60009ae4b55bfc8bef71680dc18 2024-11-21T00:17:40,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cfae7a080a894fa29c20eb43a86c028d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/cfae7a080a894fa29c20eb43a86c028d 2024-11-21T00:17:40,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d0ba87cc29124d07909deffcaa6b478d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/A/d0ba87cc29124d07909deffcaa6b478d 2024-11-21T00:17:40,693 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/0b4662f8ed2e4531b73807afa79624c2 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/0b4662f8ed2e4531b73807afa79624c2 2024-11-21T00:17:40,694 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/24bbe9c7b9c346f7b9f8159685ba7d52 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/24bbe9c7b9c346f7b9f8159685ba7d52 2024-11-21T00:17:40,695 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a05663ed563148e1b9ed019ac974f85b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/a05663ed563148e1b9ed019ac974f85b 2024-11-21T00:17:40,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f8a14bcd269b4be393117a9e684c9e2f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f8a14bcd269b4be393117a9e684c9e2f 2024-11-21T00:17:40,697 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f935edbc36e94b74af145f00a43c6a72 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/B/f935edbc36e94b74af145f00a43c6a72 2024-11-21T00:17:40,698 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/430195a4f85c4fbe995e1daf449c4d9a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/430195a4f85c4fbe995e1daf449c4d9a 2024-11-21T00:17:40,699 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/45bafc148a7148879cc04b1a30e51285 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/45bafc148a7148879cc04b1a30e51285 2024-11-21T00:17:40,700 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b337c8639161443582c2e66e4e057563 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/b337c8639161443582c2e66e4e057563 2024-11-21T00:17:40,701 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/d58d68b2b3c04da19c0c48abdba09cd3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/d58d68b2b3c04da19c0c48abdba09cd3 2024-11-21T00:17:40,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f8e2051dfa1f44398db388ddb4256ae7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/C/f8e2051dfa1f44398db388ddb4256ae7 2024-11-21T00:17:40,703 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/recovered.edits/439.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b/recovered.edits/439.seqid 2024-11-21T00:17:40,704 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/6f23b76c1c42e5c3a6984684bda2f10b 2024-11-21T00:17:40,704 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:17:40,705 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,707 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:17:40,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:17:40,709 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,709 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:17:40,709 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148260709"}]},"ts":"9223372036854775807"} 2024-11-21T00:17:40,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:17:40,711 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6f23b76c1c42e5c3a6984684bda2f10b, NAME => 'TestAcidGuarantees,,1732148233833.6f23b76c1c42e5c3a6984684bda2f10b.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:17:40,711 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:17:40,711 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148260711"}]},"ts":"9223372036854775807"} 2024-11-21T00:17:40,713 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:17:40,714 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-21T00:17:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T00:17:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:17:40,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:40,787 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=954 (was 947) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1625 (was 240) - AvailableMemoryMB LEAK? - 2024-11-21T00:17:40,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=954, ProcessCount=11, AvailableMemoryMB=1625 2024-11-21T00:17:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:17:40,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:17:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:40,798 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:17:40,798 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:40,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-21T00:17:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:17:40,799 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:17:40,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742362_1538 (size=1061) 2024-11-21T00:17:40,807 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b 2024-11-21T00:17:40,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742363_1539 (size=53) 2024-11-21T00:17:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:17:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d3ff459acb90743c62370493a30bdf44, disabling compactions & flushes 2024-11-21T00:17:41,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. after waiting 0 ms 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3ff459acb90743c62370493a30bdf44: Waiting for close lock at 1732148261213Disabling compacts and flushes for region at 1732148261213Disabling writes for close at 1732148261213Writing region close event to WAL at 1732148261213Closed at 1732148261213 2024-11-21T00:17:41,214 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:17:41,214 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148261214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148261214"}]},"ts":"1732148261214"} 2024-11-21T00:17:41,215 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:17:41,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:17:41,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148261216"}]},"ts":"1732148261216"} 2024-11-21T00:17:41,217 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:17:41,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, ASSIGN}] 2024-11-21T00:17:41,218 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, ASSIGN 2024-11-21T00:17:41,219 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=false 2024-11-21T00:17:41,369 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:41,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, ASSIGN because future has completed 2024-11-21T00:17:41,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:17:41,526 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,526 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:17:41,526 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,526 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:17:41,526 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,526 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,527 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,528 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:41,528 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName A 2024-11-21T00:17:41,528 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:41,529 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:41,529 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,530 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:41,530 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName B 2024-11-21T00:17:41,530 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:41,530 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:41,530 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,531 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:41,532 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName C 2024-11-21T00:17:41,532 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:41,532 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:41,532 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,532 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,533 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,533 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,534 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,534 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,534 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:17:41,535 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,537 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:17:41,537 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened d3ff459acb90743c62370493a30bdf44; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72832961, jitterRate=0.08529569208621979}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:17:41,537 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:41,538 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for d3ff459acb90743c62370493a30bdf44: Running coprocessor pre-open hook at 1732148261526Writing region info on filesystem at 1732148261526Initializing all the Stores at 1732148261527 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148261527Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148261527Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148261527Cleaning up temporary data from old regions at 1732148261534 (+7 ms)Running coprocessor post-open hooks at 1732148261537 (+3 ms)Region opened successfully at 1732148261538 (+1 ms) 2024-11-21T00:17:41,539 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., pid=142, masterSystemTime=1732148261523 2024-11-21T00:17:41,540 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,540 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:41,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:41,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:41,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T00:17:41,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 in 172 msec 2024-11-21T00:17:41,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-21T00:17:41,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, ASSIGN in 327 msec 2024-11-21T00:17:41,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:17:41,547 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148261547"}]},"ts":"1732148261547"} 2024-11-21T00:17:41,548 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:17:41,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:17:41,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 753 msec 2024-11-21T00:17:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:17:41,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:41,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658b2e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:41,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:41,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:41,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404254bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:41,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf10129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:41,943 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:41,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:41,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:17:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,39691,1732148113019 2024-11-21T00:17:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b14464f 2024-11-21T00:17:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:17:41,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:17:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:17:41,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:17:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:17:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742364_1540 (size=1097) 2024-11-21T00:17:42,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:17:42,363 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T00:17:42,363 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T00:17:42,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:17:42,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE}] 2024-11-21T00:17:42,369 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE 2024-11-21T00:17:42,370 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:42,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE because future has completed 2024-11-21T00:17:42,371 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:17:42,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:42,524 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,524 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:17:42,524 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing d3ff459acb90743c62370493a30bdf44, disabling compactions & flushes 2024-11-21T00:17:42,524 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,524 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,524 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. after waiting 0 ms 2024-11-21T00:17:42,524 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,528 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:17:42,529 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,529 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for d3ff459acb90743c62370493a30bdf44: Waiting for close lock at 1732148262524Running coprocessor pre-close hooks at 1732148262524Disabling compacts and flushes for region at 1732148262524Disabling writes for close at 1732148262524Writing region close event to WAL at 1732148262525 (+1 ms)Running coprocessor post-close hooks at 1732148262528 (+3 ms)Closed at 1732148262528 2024-11-21T00:17:42,529 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: d3ff459acb90743c62370493a30bdf44 to self. 2024-11-21T00:17:42,530 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,531 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=CLOSED 2024-11-21T00:17:42,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:42,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-21T00:17:42,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 in 166 msec 2024-11-21T00:17:42,540 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,40705,1732148113864; forceNewPlan=false, retain=true 2024-11-21T00:17:42,690 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=OPENING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:42,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE because future has completed 2024-11-21T00:17:42,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:17:42,847 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,847 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:17:42,847 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,847 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:17:42,847 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,847 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,848 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,848 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:42,849 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName A 2024-11-21T00:17:42,850 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:42,850 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:42,851 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,851 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:42,851 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName B 2024-11-21T00:17:42,851 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:42,852 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:42,852 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,852 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T00:17:42,853 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ff459acb90743c62370493a30bdf44 columnFamilyName C 2024-11-21T00:17:42,853 DEBUG [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:42,853 INFO [StoreOpener-d3ff459acb90743c62370493a30bdf44-1 {}] regionserver.HStore(327): Store=d3ff459acb90743c62370493a30bdf44/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:17:42,853 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,853 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,854 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,854 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,855 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,855 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,856 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:17:42,857 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,858 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened d3ff459acb90743c62370493a30bdf44; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70852298, jitterRate=0.05578151345252991}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:17:42,858 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,858 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for d3ff459acb90743c62370493a30bdf44: Running coprocessor pre-open hook at 1732148262847Writing region info on filesystem at 1732148262847Initializing all the Stores at 1732148262848 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148262848Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148262848Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148262848Cleaning up temporary data from old regions at 1732148262855 (+7 ms)Running coprocessor post-open hooks at 1732148262858 (+3 ms)Region opened successfully at 1732148262858 2024-11-21T00:17:42,859 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., pid=147, masterSystemTime=1732148262844 2024-11-21T00:17:42,861 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,861 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:42,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:42,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:17:42,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-21T00:17:42,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 in 171 msec 2024-11-21T00:17:42,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-21T00:17:42,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, REOPEN/MOVE in 497 msec 2024-11-21T00:17:42,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T00:17:42,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-21T00:17:42,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 917 msec 2024-11-21T00:17:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T00:17:42,871 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:42,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26766b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d105935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bfe98c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3875a4ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0b2156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efdbbdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,882 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0e6310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4551cc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b86ebfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613deb2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc94608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbc5abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8c8d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2123ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e85a461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7738b524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3bd43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74310a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,39691,-1 for getting cluster id 2024-11-21T00:17:42,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:17:42,900 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a60e3fdb-895f-4b66-963f-9689bc9d3dbc' 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a60e3fdb-895f-4b66-963f-9689bc9d3dbc" 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18533e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,39691,-1] 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:17:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:17:42,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:17:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28579ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:17:42,902 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,902 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,903 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,903 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,903 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,904 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-21T00:17:42,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:42,906 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,907 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,907 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:42,907 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,909 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:17:42,909 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,909 DEBUG [Thread-2422 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:42,909 DEBUG [Thread-2422 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,40705,1732148113864, seqNum=-1] 2024-11-21T00:17:42,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:17:42,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:17:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5] 2024-11-21T00:17:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:42,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:17:42,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148322982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148262978, max timestamp=1732148262984Num uniques 859; 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148262978, max timestamp=1732148262984Num uniques 859; 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148262978, max timestamp=1732148262984Num uniques 859; 2024-11-21T00:17:42,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:17:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:42,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b303ef500ca94f4a8654405a247839ac_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148262913/Put/seqid=0 2024-11-21T00:17:43,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742365_1541 (size=12154) 2024-11-21T00:17:43,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:43,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b303ef500ca94f4a8654405a247839ac_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b303ef500ca94f4a8654405a247839ac_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:43,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/1222e976f48c4d4182f9324345ca9d05, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:43,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/1222e976f48c4d4182f9324345ca9d05 is 175, key is test_row_0/A:col10/1732148262913/Put/seqid=0 2024-11-21T00:17:43,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742366_1542 (size=30955) 2024-11-21T00:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148323014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148323014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148323022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148323096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148323126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148323126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148323139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:43,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148323307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148323336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148323336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148323346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/1222e976f48c4d4182f9324345ca9d05 2024-11-21T00:17:43,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/4b4e3c99b4d74f239b7cd4b84538d79f is 50, key is test_row_0/B:col10/1732148262913/Put/seqid=0 2024-11-21T00:17:43,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742367_1543 (size=12001) 2024-11-21T00:17:43,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/4b4e3c99b4d74f239b7cd4b84538d79f 2024-11-21T00:17:43,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f64873a459dc492ca8126c0f454eb8c1 is 50, key is test_row_0/C:col10/1732148262913/Put/seqid=0 2024-11-21T00:17:43,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742368_1544 (size=12001) 2024-11-21T00:17:43,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f64873a459dc492ca8126c0f454eb8c1 2024-11-21T00:17:43,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/1222e976f48c4d4182f9324345ca9d05 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05 2024-11-21T00:17:43,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T00:17:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/4b4e3c99b4d74f239b7cd4b84538d79f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f 2024-11-21T00:17:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:43,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f64873a459dc492ca8126c0f454eb8c1 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1 2024-11-21T00:17:43,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:17:43,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d3ff459acb90743c62370493a30bdf44 in 581ms, sequenceid=15, compaction requested=false 2024-11-21T00:17:43,544 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T00:17:43,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:43,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T00:17:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:43,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c456120f42504e2788f4a6b61f4cdb62_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148263617/Put/seqid=0 2024-11-21T00:17:43,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742369_1545 (size=14594) 2024-11-21T00:17:43,631 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:43,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c456120f42504e2788f4a6b61f4cdb62_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c456120f42504e2788f4a6b61f4cdb62_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:43,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/ab10a44382e642809a252e7b2e4d65fe, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:43,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/ab10a44382e642809a252e7b2e4d65fe is 175, key is test_row_0/A:col10/1732148263617/Put/seqid=0 2024-11-21T00:17:43,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742370_1546 (size=39549) 2024-11-21T00:17:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148323646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148323646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148323650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148323657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148323757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148323967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148323980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:43,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:43,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/ab10a44382e642809a252e7b2e4d65fe 2024-11-21T00:17:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:44,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/49727c6bb033435296fe3ceeefd41f39 is 50, key is test_row_0/B:col10/1732148263617/Put/seqid=0 2024-11-21T00:17:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742371_1547 (size=12001) 2024-11-21T00:17:44,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/49727c6bb033435296fe3ceeefd41f39 2024-11-21T00:17:44,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/25e64315db4a450eb8a5c55835de15f4 is 50, key is test_row_0/C:col10/1732148263617/Put/seqid=0 2024-11-21T00:17:44,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742372_1548 (size=12001) 2024-11-21T00:17:44,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148324157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148324157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148324166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,248 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:17:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148324276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148324286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:44,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:44,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:44,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/25e64315db4a450eb8a5c55835de15f4 2024-11-21T00:17:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/ab10a44382e642809a252e7b2e4d65fe as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe 2024-11-21T00:17:44,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe, entries=200, sequenceid=51, filesize=38.6 K 2024-11-21T00:17:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/49727c6bb033435296fe3ceeefd41f39 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39 2024-11-21T00:17:44,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:17:44,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/25e64315db4a450eb8a5c55835de15f4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4 2024-11-21T00:17:44,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:17:44,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for d3ff459acb90743c62370493a30bdf44 in 889ms, sequenceid=51, compaction requested=false 2024-11-21T00:17:44,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:44,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:17:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:44,601 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182d1fb14baac4a54a79ba75130d8ce1e_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148263637/Put/seqid=0 2024-11-21T00:17:44,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742373_1549 (size=12154) 2024-11-21T00:17:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:44,650 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182d1fb14baac4a54a79ba75130d8ce1e_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d1fb14baac4a54a79ba75130d8ce1e_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/fff9917607fa45d5bfc0f4dca611d1ca, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/fff9917607fa45d5bfc0f4dca611d1ca is 175, key is test_row_0/A:col10/1732148263637/Put/seqid=0 2024-11-21T00:17:44,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742374_1550 (size=30955) 2024-11-21T00:17:44,691 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/fff9917607fa45d5bfc0f4dca611d1ca 2024-11-21T00:17:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/24f20bb5646d4f148214c221bd9b423d is 50, key is test_row_0/B:col10/1732148263637/Put/seqid=0 2024-11-21T00:17:44,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742375_1551 (size=12001) 2024-11-21T00:17:44,720 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/24f20bb5646d4f148214c221bd9b423d 2024-11-21T00:17:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/280f2af600e049c99f25914a8fb0d113 is 50, key is test_row_0/C:col10/1732148263637/Put/seqid=0 2024-11-21T00:17:44,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742376_1552 (size=12001) 2024-11-21T00:17:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:44,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:44,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148264847, max timestamp=1732148264861Num uniques 859; 2024-11-21T00:17:44,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:44,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148264847, max timestamp=1732148264861Num uniques 859; 2024-11-21T00:17:44,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:17:44,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148264847, max timestamp=1732148264861Num uniques 859; 2024-11-21T00:17:44,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:17:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148324924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148324928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148325036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148325036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148325166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,170 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148325176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,180 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148325186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,188 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,194 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/280f2af600e049c99f25914a8fb0d113 2024-11-21T00:17:45,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/fff9917607fa45d5bfc0f4dca611d1ca as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca 2024-11-21T00:17:45,214 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca, entries=150, sequenceid=61, filesize=30.2 K 2024-11-21T00:17:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/24f20bb5646d4f148214c221bd9b423d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d 2024-11-21T00:17:45,220 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T00:17:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/280f2af600e049c99f25914a8fb0d113 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113 2024-11-21T00:17:45,229 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T00:17:45,230 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.39 KB/136590 for d3ff459acb90743c62370493a30bdf44 in 629ms, sequenceid=61, compaction requested=true 2024-11-21T00:17:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-21T00:17:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-21T00:17:45,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-21T00:17:45,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3220 sec 2024-11-21T00:17:45,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3330 sec 2024-11-21T00:17:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:45,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=140.10 KB heapSize=403.78 KB 2024-11-21T00:17:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:45,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112151681cf80ece4888b969dbd1dd2cfad9_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:45,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148325306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742377_1553 (size=12154) 2024-11-21T00:17:45,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148325310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148325416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148325427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148325627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148325636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:45,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112151681cf80ece4888b969dbd1dd2cfad9_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151681cf80ece4888b969dbd1dd2cfad9_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:45,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/179a35f8b1114c50a345d8d2b692ab41, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:45,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/179a35f8b1114c50a345d8d2b692ab41 is 175, key is test_row_0/A:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:45,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742378_1554 (size=30955) 2024-11-21T00:17:45,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/179a35f8b1114c50a345d8d2b692ab41 2024-11-21T00:17:45,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e9ee786d604c4638bf33d4783bc2421f is 50, key is test_row_0/B:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:45,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742379_1555 (size=12001) 2024-11-21T00:17:45,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e9ee786d604c4638bf33d4783bc2421f 2024-11-21T00:17:45,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148325942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/879149dff3a84ae3af29c6185dabc6fc is 50, key is test_row_0/C:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:45,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148325964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:45,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742380_1556 (size=12001) 2024-11-21T00:17:45,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/879149dff3a84ae3af29c6185dabc6fc 2024-11-21T00:17:46,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/179a35f8b1114c50a345d8d2b692ab41 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41 2024-11-21T00:17:46,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41, entries=150, sequenceid=99, filesize=30.2 K 2024-11-21T00:17:46,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e9ee786d604c4638bf33d4783bc2421f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f 2024-11-21T00:17:46,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T00:17:46,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/879149dff3a84ae3af29c6185dabc6fc as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc 2024-11-21T00:17:46,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T00:17:46,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.10 KB/143460, heapSize ~403.73 KB/413424, currentSize=40.25 KB/41220 for d3ff459acb90743c62370493a30bdf44 in 797ms, sequenceid=99, compaction requested=true 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:46,048 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:46,048 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:46,054 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:46,054 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:46,054 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:46,054 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:46,054 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:46,054 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:46,055 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=129.3 K 2024-11-21T00:17:46,055 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=46.9 K 2024-11-21T00:17:46,055 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:46,055 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41] 2024-11-21T00:17:46,055 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4e3c99b4d74f239b7cd4b84538d79f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148262913 2024-11-21T00:17:46,055 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1222e976f48c4d4182f9324345ca9d05, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148262913 2024-11-21T00:17:46,055 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 49727c6bb033435296fe3ceeefd41f39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148263007 2024-11-21T00:17:46,056 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f20bb5646d4f148214c221bd9b423d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732148263637 2024-11-21T00:17:46,056 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab10a44382e642809a252e7b2e4d65fe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148263007 2024-11-21T00:17:46,057 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff9917607fa45d5bfc0f4dca611d1ca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732148263637 2024-11-21T00:17:46,057 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e9ee786d604c4638bf33d4783bc2421f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:46,057 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179a35f8b1114c50a345d8d2b692ab41, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:46,083 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:46,083 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/b25a162472054696b0c00925cadd3924 is 50, key is test_row_0/B:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:46,095 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:46,101 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121467a0356ecfe42c9982648d588cfa9d5_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:46,104 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121467a0356ecfe42c9982648d588cfa9d5_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:46,104 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121467a0356ecfe42c9982648d588cfa9d5_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:46,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742381_1557 (size=12139) 2024-11-21T00:17:46,111 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/b25a162472054696b0c00925cadd3924 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b25a162472054696b0c00925cadd3924 2024-11-21T00:17:46,118 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into b25a162472054696b0c00925cadd3924(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:46,118 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:46,118 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=12, startTime=1732148266048; duration=0sec 2024-11-21T00:17:46,118 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:46,118 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:46,118 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:46,121 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:46,121 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:46,121 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:46,121 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=46.9 K 2024-11-21T00:17:46,122 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f64873a459dc492ca8126c0f454eb8c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148262913 2024-11-21T00:17:46,122 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e64315db4a450eb8a5c55835de15f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148263007 2024-11-21T00:17:46,123 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 280f2af600e049c99f25914a8fb0d113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732148263637 2024-11-21T00:17:46,123 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 879149dff3a84ae3af29c6185dabc6fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742382_1558 (size=4469) 2024-11-21T00:17:46,138 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#472 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:46,138 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/7f1d9191f885411eb5f2842a736714e8 is 175, key is test_row_0/A:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:46,145 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:46,146 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/d1694658fb6b4676b7e1d4f00be7635b is 50, key is test_row_0/C:col10/1732148264908/Put/seqid=0 2024-11-21T00:17:46,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742383_1559 (size=31093) 2024-11-21T00:17:46,180 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/7f1d9191f885411eb5f2842a736714e8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8 2024-11-21T00:17:46,192 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 7f1d9191f885411eb5f2842a736714e8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:46,192 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:46,192 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=12, startTime=1732148266048; duration=0sec 2024-11-21T00:17:46,192 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:46,192 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:17:46,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742384_1560 (size=12139) 2024-11-21T00:17:46,231 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/d1694658fb6b4676b7e1d4f00be7635b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/d1694658fb6b4676b7e1d4f00be7635b 2024-11-21T00:17:46,245 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into d1694658fb6b4676b7e1d4f00be7635b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:46,245 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:46,245 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=12, startTime=1732148266048; duration=0sec 2024-11-21T00:17:46,245 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:46,245 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:46,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:46,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bce3a4bd17cf40e086c756322272e093_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148265265/Put/seqid=0 2024-11-21T00:17:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742385_1561 (size=14594) 2024-11-21T00:17:46,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:46,532 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bce3a4bd17cf40e086c756322272e093_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bce3a4bd17cf40e086c756322272e093_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:46,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2287deae1ced4949ad233e1ceaa6b7e6, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:46,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2287deae1ced4949ad233e1ceaa6b7e6 is 175, key is test_row_0/A:col10/1732148265265/Put/seqid=0 2024-11-21T00:17:46,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148266508, max timestamp=1732148266560Num uniques 859; 2024-11-21T00:17:46,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742386_1562 (size=39549) 2024-11-21T00:17:46,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148266508, max timestamp=1732148266560Num uniques 859; 2024-11-21T00:17:46,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:46,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148266508, max timestamp=1732148266560Num uniques 859; 2024-11-21T00:17:46,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:17:46,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2287deae1ced4949ad233e1ceaa6b7e6 2024-11-21T00:17:46,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e71a12e988e645a6bc8d7e337b06e110 is 50, key is test_row_0/B:col10/1732148265265/Put/seqid=0 2024-11-21T00:17:46,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742387_1563 (size=12001) 2024-11-21T00:17:46,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e71a12e988e645a6bc8d7e337b06e110 2024-11-21T00:17:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148326670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:46,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148326674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:46,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/ce25b3128bba46bfb7bfe515a1c77c81 is 50, key is test_row_0/C:col10/1732148265265/Put/seqid=0 2024-11-21T00:17:46,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742388_1564 (size=12001) 2024-11-21T00:17:46,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148326786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148326786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148326998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148326998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:17:47,069 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:47,071 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-21T00:17:47,074 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:47,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:47,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:47,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/ce25b3128bba46bfb7bfe515a1c77c81 2024-11-21T00:17:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2287deae1ced4949ad233e1ceaa6b7e6 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6 2024-11-21T00:17:47,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6, entries=200, sequenceid=112, filesize=38.6 K 2024-11-21T00:17:47,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e71a12e988e645a6bc8d7e337b06e110 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110 2024-11-21T00:17:47,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T00:17:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/ce25b3128bba46bfb7bfe515a1c77c81 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81 2024-11-21T00:17:47,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T00:17:47,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d3ff459acb90743c62370493a30bdf44 in 699ms, sequenceid=112, compaction requested=false 2024-11-21T00:17:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:47,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732148266644, max timestamp=1732148267197Num uniques 1018; 2024-11-21T00:17:47,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T00:17:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:47,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=109.58 KB heapSize=312.13 KB 2024-11-21T00:17:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:17:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:47,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:47,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:47,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T00:17:47,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c8b45ea01bf4c44b21a2a42826d8ad6_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:47,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148327238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742389_1565 (size=17284) 2024-11-21T00:17:47,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:47,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c8b45ea01bf4c44b21a2a42826d8ad6_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c8b45ea01bf4c44b21a2a42826d8ad6_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:47,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148327243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148327244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3f5a6ec82a5d48fda0a67555ce68b3ea, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:47,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3f5a6ec82a5d48fda0a67555ce68b3ea is 175, key is test_row_0/A:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742390_1566 (size=48389) 2024-11-21T00:17:47,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148327317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148327317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148327346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148327358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148327358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:47,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148327557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148327567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148327567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3f5a6ec82a5d48fda0a67555ce68b3ea 2024-11-21T00:17:47,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/125a4f9041854db7837a8ab9ee5561ab is 50, key is test_row_0/B:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:47,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742391_1567 (size=12151) 2024-11-21T00:17:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148327826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148327837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:47,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148327868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148327885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148327885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/125a4f9041854db7837a8ab9ee5561ab 2024-11-21T00:17:48,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/eedef729b48d472bad474b8a35ba8459 is 50, key is test_row_0/C:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:48,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:48,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:48,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:48,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742392_1568 (size=12151) 2024-11-21T00:17:48,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/eedef729b48d472bad474b8a35ba8459 2024-11-21T00:17:48,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3f5a6ec82a5d48fda0a67555ce68b3ea as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea 2024-11-21T00:17:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:48,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea, entries=250, sequenceid=151, filesize=47.3 K 2024-11-21T00:17:48,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/125a4f9041854db7837a8ab9ee5561ab as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab 2024-11-21T00:17:48,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T00:17:48,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/eedef729b48d472bad474b8a35ba8459 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459 2024-11-21T00:17:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T00:17:48,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~312.08 KB/319568, currentSize=73.80 KB/75570 for d3ff459acb90743c62370493a30bdf44 in 1060ms, sequenceid=151, compaction requested=true 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:48,264 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:48,264 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:48,266 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:48,266 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:48,266 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:48,266 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:48,266 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,266 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=116.2 K 2024-11-21T00:17:48,266 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,266 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea] 2024-11-21T00:17:48,266 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,266 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b25a162472054696b0c00925cadd3924, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=35.4 K 2024-11-21T00:17:48,267 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1d9191f885411eb5f2842a736714e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:48,267 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting b25a162472054696b0c00925cadd3924, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:48,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e71a12e988e645a6bc8d7e337b06e110, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732148265265 2024-11-21T00:17:48,268 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2287deae1ced4949ad233e1ceaa6b7e6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732148265265 2024-11-21T00:17:48,268 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5a6ec82a5d48fda0a67555ce68b3ea, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266644 2024-11-21T00:17:48,268 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 125a4f9041854db7837a8ab9ee5561ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266666 2024-11-21T00:17:48,276 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:48,278 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/754f273660ac41268231787f6b29f324 is 50, key is test_row_0/B:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:48,281 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:48,308 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112188149ecbeecc4b12a34963001344e56d_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:48,311 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112188149ecbeecc4b12a34963001344e56d_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:48,311 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112188149ecbeecc4b12a34963001344e56d_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:48,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:17:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,320 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742393_1569 (size=12391) 2024-11-21T00:17:48,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742394_1570 (size=4469) 2024-11-21T00:17:48,389 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#481 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:48,389 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53471ca54a41462d81b8090a33499780 is 175, key is test_row_0/A:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:48,390 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/754f273660ac41268231787f6b29f324 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/754f273660ac41268231787f6b29f324 2024-11-21T00:17:48,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112144f0495289754746adec01cfeded99b1_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148267229/Put/seqid=0 2024-11-21T00:17:48,403 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into 754f273660ac41268231787f6b29f324(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:48,403 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:48,403 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=13, startTime=1732148268264; duration=0sec 2024-11-21T00:17:48,403 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:48,403 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:48,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:48,403 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:48,405 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:48,405 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:48,405 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:48,405 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/d1694658fb6b4676b7e1d4f00be7635b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=35.4 K 2024-11-21T00:17:48,411 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting d1694658fb6b4676b7e1d4f00be7635b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732148264908 2024-11-21T00:17:48,412 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting ce25b3128bba46bfb7bfe515a1c77c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732148265265 2024-11-21T00:17:48,414 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting eedef729b48d472bad474b8a35ba8459, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266666 2024-11-21T00:17:48,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742396_1572 (size=12304) 2024-11-21T00:17:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742395_1571 (size=31345) 2024-11-21T00:17:48,437 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:48,437 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/fba1ef8378d7411386287bfa1c776e41 is 50, key is test_row_0/C:col10/1732148267203/Put/seqid=0 2024-11-21T00:17:48,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742397_1573 (size=12391) 2024-11-21T00:17:48,451 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/fba1ef8378d7411386287bfa1c776e41 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/fba1ef8378d7411386287bfa1c776e41 2024-11-21T00:17:48,500 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into fba1ef8378d7411386287bfa1c776e41(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:48,501 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:48,501 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=13, startTime=1732148268264; duration=0sec 2024-11-21T00:17:48,501 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:48,501 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148328489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148328503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148328503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148328608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148328626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148328628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148328817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148328838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:48,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148328840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148328847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,854 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,856 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53471ca54a41462d81b8090a33499780 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780 2024-11-21T00:17:48,861 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112144f0495289754746adec01cfeded99b1_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144f0495289754746adec01cfeded99b1_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:48,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2dc549bb017544e481d40ac3d47f31b9, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:48,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:48,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2dc549bb017544e481d40ac3d47f31b9 is 175, key is test_row_0/A:col10/1732148267229/Put/seqid=0 2024-11-21T00:17:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148328857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:48,867 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:48,877 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 53471ca54a41462d81b8090a33499780(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:48,877 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:48,877 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=13, startTime=1732148268264; duration=0sec 2024-11-21T00:17:48,877 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:48,877 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:17:48,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742398_1574 (size=31105) 2024-11-21T00:17:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148329128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148329146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148329157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:49,303 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2dc549bb017544e481d40ac3d47f31b9 2024-11-21T00:17:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/932497b3d1f744568e4b1f89c875a0b0 is 50, key is test_row_0/B:col10/1732148267229/Put/seqid=0 2024-11-21T00:17:49,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742399_1575 (size=12151) 2024-11-21T00:17:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148329657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148329666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148329680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:49,775 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/932497b3d1f744568e4b1f89c875a0b0 2024-11-21T00:17:49,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e8af27846f6f49fd85ecd35202e56b42 is 50, key is test_row_0/C:col10/1732148267229/Put/seqid=0 2024-11-21T00:17:49,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742400_1576 (size=12151) 2024-11-21T00:17:49,796 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e8af27846f6f49fd85ecd35202e56b42 2024-11-21T00:17:49,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/2dc549bb017544e481d40ac3d47f31b9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9 2024-11-21T00:17:49,819 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9, entries=150, sequenceid=165, filesize=30.4 K 2024-11-21T00:17:49,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/932497b3d1f744568e4b1f89c875a0b0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0 2024-11-21T00:17:49,832 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0, entries=150, sequenceid=165, filesize=11.9 K 2024-11-21T00:17:49,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e8af27846f6f49fd85ecd35202e56b42 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42 2024-11-21T00:17:49,838 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42, entries=150, sequenceid=165, filesize=11.9 K 2024-11-21T00:17:49,839 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d3ff459acb90743c62370493a30bdf44 in 1519ms, sequenceid=165, compaction requested=false 2024-11-21T00:17:49,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:49,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:49,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-21T00:17:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-21T00:17:49,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T00:17:49,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7680 sec 2024-11-21T00:17:49,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7760 sec 2024-11-21T00:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:50,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dafdb116dc9741d09e7d2ffdb7cd97f2_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148330698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742401_1577 (size=14794) 2024-11-21T00:17:50,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148330702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148330702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:50,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dafdb116dc9741d09e7d2ffdb7cd97f2_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dafdb116dc9741d09e7d2ffdb7cd97f2_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:50,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/238f0ee52732434ab858b6f278744455, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/238f0ee52732434ab858b6f278744455 is 175, key is test_row_0/A:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742402_1578 (size=39749) 2024-11-21T00:17:50,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/238f0ee52732434ab858b6f278744455 2024-11-21T00:17:50,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e8253462b47d4b39bf72c7552e9dbd8f is 50, key is test_row_0/B:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742403_1579 (size=12151) 2024-11-21T00:17:50,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e8253462b47d4b39bf72c7552e9dbd8f 2024-11-21T00:17:50,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/da5e89a548b140c481baf8a4a73e1f19 is 50, key is test_row_0/C:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148330810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742404_1580 (size=12151) 2024-11-21T00:17:50,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/da5e89a548b140c481baf8a4a73e1f19 2024-11-21T00:17:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148330816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/238f0ee52732434ab858b6f278744455 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455 2024-11-21T00:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148330824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455, entries=200, sequenceid=188, filesize=38.8 K 2024-11-21T00:17:50,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e8253462b47d4b39bf72c7552e9dbd8f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f 2024-11-21T00:17:50,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f, entries=150, sequenceid=188, filesize=11.9 K 2024-11-21T00:17:50,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/da5e89a548b140c481baf8a4a73e1f19 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19 2024-11-21T00:17:50,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19, entries=150, sequenceid=188, filesize=11.9 K 2024-11-21T00:17:50,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for d3ff459acb90743c62370493a30bdf44 in 174ms, sequenceid=188, compaction requested=true 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:50,844 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:50,844 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:50,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:50,845 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:50,845 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:50,846 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:50,846 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:50,846 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:50,846 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:50,846 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=99.8 K 2024-11-21T00:17:50,846 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/754f273660ac41268231787f6b29f324, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=35.8 K 2024-11-21T00:17:50,846 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:50,846 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455] 2024-11-21T00:17:50,846 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 754f273660ac41268231787f6b29f324, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266666 2024-11-21T00:17:50,846 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53471ca54a41462d81b8090a33499780, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266666 2024-11-21T00:17:50,847 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 932497b3d1f744568e4b1f89c875a0b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732148267229 2024-11-21T00:17:50,847 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dc549bb017544e481d40ac3d47f31b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732148267229 2024-11-21T00:17:50,847 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e8253462b47d4b39bf72c7552e9dbd8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268442 2024-11-21T00:17:50,847 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238f0ee52732434ab858b6f278744455, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268433 2024-11-21T00:17:50,868 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:50,868 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e70578e351094cea8c5666736ae29c23 is 50, key is test_row_0/B:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,877 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:50,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:50,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:50,896 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211f14c9391104432cb2900bd1c9b30efc_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,904 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211f14c9391104432cb2900bd1c9b30efc_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,904 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f14c9391104432cb2900bd1c9b30efc_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742406_1582 (size=4469) 2024-11-21T00:17:50,910 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#490 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:50,911 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/142d1d5e70a14696b649138125f2af8d is 175, key is test_row_0/A:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742405_1581 (size=12493) 2024-11-21T00:17:50,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211bec41f4883745368354437188075e5c_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148270877/Put/seqid=0 2024-11-21T00:17:50,928 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e70578e351094cea8c5666736ae29c23 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e70578e351094cea8c5666736ae29c23 2024-11-21T00:17:50,933 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into e70578e351094cea8c5666736ae29c23(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:50,933 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:50,934 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=13, startTime=1732148270844; duration=0sec 2024-11-21T00:17:50,934 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:50,934 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:50,934 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:50,935 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:50,935 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:50,935 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:50,935 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/fba1ef8378d7411386287bfa1c776e41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=35.8 K 2024-11-21T00:17:50,937 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fba1ef8378d7411386287bfa1c776e41, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732148266666 2024-11-21T00:17:50,937 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e8af27846f6f49fd85ecd35202e56b42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732148267229 2024-11-21T00:17:50,938 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting da5e89a548b140c481baf8a4a73e1f19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268442 2024-11-21T00:17:50,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742408_1584 (size=12304) 2024-11-21T00:17:50,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:50,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211bec41f4883745368354437188075e5c_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211bec41f4883745368354437188075e5c_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:50,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/69784d1a237040d0880d1fd791f2235f, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:50,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/69784d1a237040d0880d1fd791f2235f is 175, key is test_row_0/A:col10/1732148270877/Put/seqid=0 2024-11-21T00:17:50,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148330957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148330958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:50,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742407_1583 (size=31447) 2024-11-21T00:17:50,966 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:50,967 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/70234187923b42f9bebb29b69a90f70b is 50, key is test_row_0/C:col10/1732148268490/Put/seqid=0 2024-11-21T00:17:51,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742409_1585 (size=31105) 2024-11-21T00:17:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148331017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148331026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148331040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742410_1586 (size=12493) 2024-11-21T00:17:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148331066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148331078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:17:51,233 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:51,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-21T00:17:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:51,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:51,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:51,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148331280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148331327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148331336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148331357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,378 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/142d1d5e70a14696b649138125f2af8d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d 2024-11-21T00:17:51,393 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 142d1d5e70a14696b649138125f2af8d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:51,393 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:51,393 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=13, startTime=1732148270844; duration=0sec 2024-11-21T00:17:51,393 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:51,393 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:17:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:17:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/69784d1a237040d0880d1fd791f2235f 2024-11-21T00:17:51,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/45e28e9ab4ed42aca260a4714b36bee9 is 50, key is test_row_0/B:col10/1732148270877/Put/seqid=0 2024-11-21T00:17:51,458 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/70234187923b42f9bebb29b69a90f70b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/70234187923b42f9bebb29b69a90f70b 2024-11-21T00:17:51,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742411_1587 (size=12151) 2024-11-21T00:17:51,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/45e28e9ab4ed42aca260a4714b36bee9 2024-11-21T00:17:51,470 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into 70234187923b42f9bebb29b69a90f70b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:51,472 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:51,472 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=13, startTime=1732148270844; duration=0sec 2024-11-21T00:17:51,472 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:51,472 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:51,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/bb1b7faf47da4249af3c6f4db9cae38d is 50, key is test_row_0/C:col10/1732148270877/Put/seqid=0 2024-11-21T00:17:51,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742412_1588 (size=12151) 2024-11-21T00:17:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:51,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:17:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148331596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148331596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:17:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148331846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148331846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148331866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:51,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:17:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:51,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:51,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/bb1b7faf47da4249af3c6f4db9cae38d 2024-11-21T00:17:51,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/69784d1a237040d0880d1fd791f2235f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f 2024-11-21T00:17:51,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f, entries=150, sequenceid=202, filesize=30.4 K 2024-11-21T00:17:51,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/45e28e9ab4ed42aca260a4714b36bee9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9 2024-11-21T00:17:51,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T00:17:51,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/bb1b7faf47da4249af3c6f4db9cae38d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d 2024-11-21T00:17:51,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T00:17:51,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d3ff459acb90743c62370493a30bdf44 in 1030ms, sequenceid=202, compaction requested=false 2024-11-21T00:17:51,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:52,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:17:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:52,023 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211952f266e8dd40baa66714b2fb477328_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148270956/Put/seqid=0 2024-11-21T00:17:52,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742413_1589 (size=12304) 2024-11-21T00:17:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148332138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148332140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148332256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148332256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148332466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148332467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:52,477 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211952f266e8dd40baa66714b2fb477328_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211952f266e8dd40baa66714b2fb477328_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e311080224ad426d834b0f1cbc645682, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e311080224ad426d834b0f1cbc645682 is 175, key is test_row_0/A:col10/1732148270956/Put/seqid=0 2024-11-21T00:17:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742414_1590 (size=31105) 2024-11-21T00:17:52,491 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e311080224ad426d834b0f1cbc645682 2024-11-21T00:17:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/df35e800f56d43b290a625c9b73e6f98 is 50, key is test_row_0/B:col10/1732148270956/Put/seqid=0 2024-11-21T00:17:52,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742415_1591 (size=12151) 2024-11-21T00:17:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148332777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148332777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148332857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,860 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148332866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,869 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_0' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148332888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:52,895 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:52,911 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/df35e800f56d43b290a625c9b73e6f98 2024-11-21T00:17:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/3a04d5bd9aed405881a4be1d83bf92ed is 50, key is test_row_0/C:col10/1732148270956/Put/seqid=0 2024-11-21T00:17:52,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742416_1592 (size=12151) 2024-11-21T00:17:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/3a04d5bd9aed405881a4be1d83bf92ed 2024-11-21T00:17:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e311080224ad426d834b0f1cbc645682 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682 2024-11-21T00:17:52,975 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682, entries=150, sequenceid=223, filesize=30.4 K 2024-11-21T00:17:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/df35e800f56d43b290a625c9b73e6f98 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98 2024-11-21T00:17:52,982 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T00:17:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/3a04d5bd9aed405881a4be1d83bf92ed as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed 2024-11-21T00:17:52,993 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T00:17:52,993 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d3ff459acb90743c62370493a30bdf44 in 970ms, sequenceid=223, compaction requested=true 2024-11-21T00:17:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-21T00:17:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-21T00:17:52,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-21T00:17:52,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7590 sec 2024-11-21T00:17:53,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7660 sec 2024-11-21T00:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:53,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:53,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121827a67b9479a4b0a98101a87c3a8943a_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:53,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742417_1593 (size=14794) 2024-11-21T00:17:53,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148333335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148333336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:17:53,387 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:53,387 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T00:17:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:53,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:53,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:53,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:53,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148333446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148333446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:53,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148333656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148333656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:53,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:53,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121827a67b9479a4b0a98101a87c3a8943a_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121827a67b9479a4b0a98101a87c3a8943a_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:53,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/236ae58649df4356b2316b8a7bd419e5, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:53,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/236ae58649df4356b2316b8a7bd419e5 is 175, key is test_row_0/A:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:53,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742418_1594 (size=39749) 2024-11-21T00:17:53,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148333966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148333967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:53,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:53,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:54,133 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/236ae58649df4356b2316b8a7bd419e5 2024-11-21T00:17:54,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/52f54f3530bd4ce99c05d0e3bc89feb0 is 50, key is test_row_0/B:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:54,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:54,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742419_1595 (size=12151) 2024-11-21T00:17:54,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148334477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148334478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:54,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/52f54f3530bd4ce99c05d0e3bc89feb0 2024-11-21T00:17:54,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/b7b97bc9c3cd46a8b5676f976aad1f1e is 50, key is test_row_0/C:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:54,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742420_1596 (size=12151) 2024-11-21T00:17:54,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/b7b97bc9c3cd46a8b5676f976aad1f1e 2024-11-21T00:17:54,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/236ae58649df4356b2316b8a7bd419e5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5 2024-11-21T00:17:54,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5, entries=200, sequenceid=239, filesize=38.8 K 2024-11-21T00:17:54,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/52f54f3530bd4ce99c05d0e3bc89feb0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0 2024-11-21T00:17:54,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T00:17:54,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/b7b97bc9c3cd46a8b5676f976aad1f1e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e 2024-11-21T00:17:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T00:17:54,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d3ff459acb90743c62370493a30bdf44 in 1382ms, sequenceid=239, compaction requested=true 2024-11-21T00:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:54,672 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:54,672 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:54,673 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:54,674 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:54,674 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,674 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=130.3 K 2024-11-21T00:17:54,674 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,674 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5] 2024-11-21T00:17:54,674 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:54,674 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:54,674 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,674 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e70578e351094cea8c5666736ae29c23, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=47.8 K 2024-11-21T00:17:54,675 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70578e351094cea8c5666736ae29c23, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268442 2024-11-21T00:17:54,675 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 142d1d5e70a14696b649138125f2af8d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268442 2024-11-21T00:17:54,675 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e28e9ab4ed42aca260a4714b36bee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732148270691 2024-11-21T00:17:54,676 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting df35e800f56d43b290a625c9b73e6f98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148270946 2024-11-21T00:17:54,676 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 69784d1a237040d0880d1fd791f2235f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732148270691 2024-11-21T00:17:54,676 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f54f3530bd4ce99c05d0e3bc89feb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:54,676 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e311080224ad426d834b0f1cbc645682, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148270946 2024-11-21T00:17:54,677 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 236ae58649df4356b2316b8a7bd419e5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:54,702 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:54,702 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/01d9c759dcc84d7b9ae0a96b49a2189b is 50, key is test_row_0/B:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:54,714 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:54,739 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112165c4e3870c844cbba83a5eb40a072c36_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:54,742 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112165c4e3870c844cbba83a5eb40a072c36_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:54,742 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112165c4e3870c844cbba83a5eb40a072c36_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:54,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742421_1597 (size=12629) 2024-11-21T00:17:54,763 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/01d9c759dcc84d7b9ae0a96b49a2189b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/01d9c759dcc84d7b9ae0a96b49a2189b 2024-11-21T00:17:54,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742422_1598 (size=4469) 2024-11-21T00:17:54,769 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#502 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:54,769 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/db001f46ac684224bde214deb0efa4d3 is 175, key is test_row_0/A:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:54,774 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into 01d9c759dcc84d7b9ae0a96b49a2189b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:54,774 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:54,774 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=12, startTime=1732148274672; duration=0sec 2024-11-21T00:17:54,774 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:54,774 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:54,774 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:17:54,776 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:17:54,776 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:54,777 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,777 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/70234187923b42f9bebb29b69a90f70b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=47.8 K 2024-11-21T00:17:54,777 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70234187923b42f9bebb29b69a90f70b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732148268442 2024-11-21T00:17:54,777 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1b7faf47da4249af3c6f4db9cae38d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732148270691 2024-11-21T00:17:54,778 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a04d5bd9aed405881a4be1d83bf92ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148270946 2024-11-21T00:17:54,778 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b97bc9c3cd46a8b5676f976aad1f1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:54,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:17:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:54,780 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:54,819 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:54,819 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/22dccf87bbe64faaba1318e27eacaad9 is 50, key is test_row_0/C:col10/1732148273290/Put/seqid=0 2024-11-21T00:17:54,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742423_1599 (size=31583) 2024-11-21T00:17:54,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212d64d2dad5054be1b7cabede3547da5f_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148273335/Put/seqid=0 2024-11-21T00:17:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742424_1600 (size=12629) 2024-11-21T00:17:54,841 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/db001f46ac684224bde214deb0efa4d3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3 2024-11-21T00:17:54,845 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/22dccf87bbe64faaba1318e27eacaad9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/22dccf87bbe64faaba1318e27eacaad9 2024-11-21T00:17:54,850 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into db001f46ac684224bde214deb0efa4d3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:54,850 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:54,850 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=12, startTime=1732148274672; duration=0sec 2024-11-21T00:17:54,850 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:54,850 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:17:54,851 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into 22dccf87bbe64faaba1318e27eacaad9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:54,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:54,851 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=12, startTime=1732148274673; duration=0sec 2024-11-21T00:17:54,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:54,851 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:54,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742425_1601 (size=12304) 2024-11-21T00:17:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212d64d2dad5054be1b7cabede3547da5f_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212d64d2dad5054be1b7cabede3547da5f_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/567ca04acb5245c59ea7fe42329e134d, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/567ca04acb5245c59ea7fe42329e134d is 175, key is test_row_0/A:col10/1732148273335/Put/seqid=0 2024-11-21T00:17:54,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:54,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742426_1602 (size=31105) 2024-11-21T00:17:54,898 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/567ca04acb5245c59ea7fe42329e134d 2024-11-21T00:17:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/02a4e4286b3a48818ae73e2fc26d5797 is 50, key is test_row_0/B:col10/1732148273335/Put/seqid=0 2024-11-21T00:17:54,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148334928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:54,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742427_1603 (size=12151) 2024-11-21T00:17:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148334935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148334936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:54,944 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/02a4e4286b3a48818ae73e2fc26d5797 2024-11-21T00:17:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/773b15205bb84ad5a0f247c729ae0b87 is 50, key is test_row_0/C:col10/1732148273335/Put/seqid=0 2024-11-21T00:17:55,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742428_1604 (size=12151) 2024-11-21T00:17:55,031 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/773b15205bb84ad5a0f247c729ae0b87 2024-11-21T00:17:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/567ca04acb5245c59ea7fe42329e134d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d 2024-11-21T00:17:55,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148335046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,051 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d, entries=150, sequenceid=256, filesize=30.4 K 2024-11-21T00:17:55,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148335047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148335047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/02a4e4286b3a48818ae73e2fc26d5797 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797 2024-11-21T00:17:55,063 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T00:17:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/773b15205bb84ad5a0f247c729ae0b87 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87 2024-11-21T00:17:55,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T00:17:55,075 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d3ff459acb90743c62370493a30bdf44 in 295ms, sequenceid=256, compaction requested=false 2024-11-21T00:17:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T00:17:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T00:17:55,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T00:17:55,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-21T00:17:55,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.6920 sec 2024-11-21T00:17:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:55,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:55,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a001e706be3e45488fe780c76793f5ea_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742429_1605 (size=14994) 2024-11-21T00:17:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148335289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148335295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148335295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148335407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148335407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148335407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148335487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148335487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,490 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,491 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:17:55,537 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:55,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T00:17:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:55,540 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:55,540 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:55,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:55,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148335616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148335616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148335616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:55,670 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:55,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a001e706be3e45488fe780c76793f5ea_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a001e706be3e45488fe780c76793f5ea_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:55,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/0355f20c429543ea93299130aaffd00a, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:55,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/0355f20c429543ea93299130aaffd00a is 175, key is test_row_0/A:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:55,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:55,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742430_1606 (size=39949) 2024-11-21T00:17:55,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:55,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:55,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:55,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:55,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148335926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148335926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148335926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:56,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/0355f20c429543ea93299130aaffd00a 2024-11-21T00:17:56,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/cd404ba0889148b3816f40791bf0c08d is 50, key is test_row_0/B:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:56,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742431_1607 (size=12301) 2024-11-21T00:17:56,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:56,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148336437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148336437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148336437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:56,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/cd404ba0889148b3816f40791bf0c08d 2024-11-21T00:17:56,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f5019c13f05a45c89f6b68e851c46a56 is 50, key is test_row_0/C:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:56,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742432_1608 (size=12301) 2024-11-21T00:17:56,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:56,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:17:56,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f5019c13f05a45c89f6b68e851c46a56 2024-11-21T00:17:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/0355f20c429543ea93299130aaffd00a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a 2024-11-21T00:17:56,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a, entries=200, sequenceid=275, filesize=39.0 K 2024-11-21T00:17:56,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/cd404ba0889148b3816f40791bf0c08d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d 2024-11-21T00:17:56,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T00:17:56,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/f5019c13f05a45c89f6b68e851c46a56 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56 2024-11-21T00:17:56,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T00:17:56,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3ff459acb90743c62370493a30bdf44 in 1731ms, sequenceid=275, compaction requested=true 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:56,989 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:56,989 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:56,991 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:56,991 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:56,992 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,992 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=100.2 K 2024-11-21T00:17:56,992 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,992 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a] 2024-11-21T00:17:56,992 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:56,992 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:56,992 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:56,992 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/01d9c759dcc84d7b9ae0a96b49a2189b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.2 K 2024-11-21T00:17:56,997 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting db001f46ac684224bde214deb0efa4d3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:56,997 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d9c759dcc84d7b9ae0a96b49a2189b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:56,997 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a4e4286b3a48818ae73e2fc26d5797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732148273326 2024-11-21T00:17:56,997 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567ca04acb5245c59ea7fe42329e134d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732148273326 2024-11-21T00:17:56,998 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting cd404ba0889148b3816f40791bf0c08d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:56,998 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0355f20c429543ea93299130aaffd00a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:57,021 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:57,026 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:57,027 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/824a7c69182541dcb9c087f4e1de96c8 is 50, key is test_row_0/B:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:57,041 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d5d0dfa0cadf437fab8d540657bb1dda_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:57,043 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d5d0dfa0cadf437fab8d540657bb1dda_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:57,043 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d5d0dfa0cadf437fab8d540657bb1dda_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:57,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:17:57,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:57,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:57,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742433_1609 (size=12881) 2024-11-21T00:17:57,094 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/824a7c69182541dcb9c087f4e1de96c8 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/824a7c69182541dcb9c087f4e1de96c8 2024-11-21T00:17:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742434_1610 (size=4469) 2024-11-21T00:17:57,102 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into 824a7c69182541dcb9c087f4e1de96c8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:57,103 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:57,103 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=13, startTime=1732148276989; duration=0sec 2024-11-21T00:17:57,103 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:57,103 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:57,103 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:57,105 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:57,106 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:57,106 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:57,106 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/22dccf87bbe64faaba1318e27eacaad9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.2 K 2024-11-21T00:17:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e55318d06cbf4e1daeff93a4b42ef8b5_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148275274/Put/seqid=0 2024-11-21T00:17:57,108 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 22dccf87bbe64faaba1318e27eacaad9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732148272126 2024-11-21T00:17:57,108 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 773b15205bb84ad5a0f247c729ae0b87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732148273326 2024-11-21T00:17:57,109 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting f5019c13f05a45c89f6b68e851c46a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:57,109 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#510 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:57,109 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/6eebfc379c29470eb88c70d3d4ce150d is 175, key is test_row_0/A:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742436_1612 (size=31835) 2024-11-21T00:17:57,117 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:57,118 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/77f9cef6c54a4abba3f17ade36ff53d9 is 50, key is test_row_0/C:col10/1732148274903/Put/seqid=0 2024-11-21T00:17:57,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742435_1611 (size=12454) 2024-11-21T00:17:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742437_1613 (size=12881) 2024-11-21T00:17:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:17:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148337483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148337488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148337490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148337501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,503 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_2' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:57,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148337507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,509 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,40705,1732148113864 for 'test_row_1' in d3ff459acb90743c62370493a30bdf44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:17:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,519 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/6eebfc379c29470eb88c70d3d4ce150d as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d 2024-11-21T00:17:57,524 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 6eebfc379c29470eb88c70d3d4ce150d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:57,524 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:57,524 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=13, startTime=1732148276989; duration=0sec 2024-11-21T00:17:57,524 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:57,524 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:17:57,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:57,531 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e55318d06cbf4e1daeff93a4b42ef8b5_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e55318d06cbf4e1daeff93a4b42ef8b5_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/02fe41801b354d5181d882d85506e506, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/02fe41801b354d5181d882d85506e506 is 175, key is test_row_0/A:col10/1732148275274/Put/seqid=0 2024-11-21T00:17:57,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742438_1614 (size=31255) 2024-11-21T00:17:57,539 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/02fe41801b354d5181d882d85506e506 2024-11-21T00:17:57,539 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/77f9cef6c54a4abba3f17ade36ff53d9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/77f9cef6c54a4abba3f17ade36ff53d9 2024-11-21T00:17:57,548 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into 77f9cef6c54a4abba3f17ade36ff53d9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:57,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:57,548 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=13, startTime=1732148276989; duration=0sec 2024-11-21T00:17:57,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:57,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/1b74f9449954471da011be0261e6c3fd is 50, key is test_row_0/B:col10/1732148275274/Put/seqid=0 2024-11-21T00:17:57,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742439_1615 (size=12301) 2024-11-21T00:17:57,560 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/1b74f9449954471da011be0261e6c3fd 2024-11-21T00:17:57,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e318522cc7bb4800bb8f23b3bd97d9d3 is 50, key is test_row_0/C:col10/1732148275274/Put/seqid=0 2024-11-21T00:17:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742440_1616 (size=12301) 2024-11-21T00:17:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148337597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148337606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148337607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148337808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148337817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148337817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:57,976 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e318522cc7bb4800bb8f23b3bd97d9d3 2024-11-21T00:17:57,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/02fe41801b354d5181d882d85506e506 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506 2024-11-21T00:17:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506, entries=150, sequenceid=290, filesize=30.5 K 2024-11-21T00:17:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/1b74f9449954471da011be0261e6c3fd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd 2024-11-21T00:17:57,987 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:17:57,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/e318522cc7bb4800bb8f23b3bd97d9d3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3 2024-11-21T00:17:57,991 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:17:57,991 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d3ff459acb90743c62370493a30bdf44 in 917ms, sequenceid=290, compaction requested=false 2024-11-21T00:17:57,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:57,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:57,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T00:17:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T00:17:57,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T00:17:57,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4520 sec 2024-11-21T00:17:57,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.4590 sec 2024-11-21T00:17:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:58,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T00:17:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:58,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:58,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:58,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fb1f57eb9ebe4b298538f234026834ec_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148338147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148338147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148338148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742441_1617 (size=12454) 2024-11-21T00:17:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148338257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148338257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148338266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148338467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148338467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148338476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:58,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fb1f57eb9ebe4b298538f234026834ec_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fb1f57eb9ebe4b298538f234026834ec_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:58,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c64806d7fce54f7fb964792c76817f80, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:58,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c64806d7fce54f7fb964792c76817f80 is 175, key is test_row_0/A:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:58,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742442_1618 (size=31255) 2024-11-21T00:17:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148338777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148338786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148338786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:58,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c64806d7fce54f7fb964792c76817f80 2024-11-21T00:17:59,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/fbe1c68690f14d05a4f4c720669c04dd is 50, key is test_row_0/B:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:59,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742443_1619 (size=12301) 2024-11-21T00:17:59,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/fbe1c68690f14d05a4f4c720669c04dd 2024-11-21T00:17:59,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/a8dbc9a66df14f4d9b9b3612482ee4c5 is 50, key is test_row_0/C:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:59,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742444_1620 (size=12301) 2024-11-21T00:17:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148339286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:59,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148339297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:59,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:17:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148339304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:17:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:17:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:17:59,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/a8dbc9a66df14f4d9b9b3612482ee4c5 2024-11-21T00:17:59,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c64806d7fce54f7fb964792c76817f80 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80 2024-11-21T00:17:59,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80, entries=150, sequenceid=311, filesize=30.5 K 2024-11-21T00:17:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/fbe1c68690f14d05a4f4c720669c04dd as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd 2024-11-21T00:17:59,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T00:17:59,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/a8dbc9a66df14f4d9b9b3612482ee4c5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5 2024-11-21T00:17:59,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T00:17:59,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d3ff459acb90743c62370493a30bdf44 in 1421ms, sequenceid=311, compaction requested=true 2024-11-21T00:17:59,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:59,541 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:59,541 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:59,541 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:17:59,542 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:59,542 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=92.1 K 2024-11-21T00:17:59,542 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:59,542 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80] 2024-11-21T00:17:59,542 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eebfc379c29470eb88c70d3d4ce150d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:59,542 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02fe41801b354d5181d882d85506e506, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148275274 2024-11-21T00:17:59,542 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64806d7fce54f7fb964792c76817f80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:17:59,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:17:59,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:59,547 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:17:59,548 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:59,548 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/824a7c69182541dcb9c087f4e1de96c8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.6 K 2024-11-21T00:17:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 824a7c69182541dcb9c087f4e1de96c8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:59,548 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b74f9449954471da011be0261e6c3fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148275274 2024-11-21T00:17:59,549 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe1c68690f14d05a4f4c720669c04dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:17:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:17:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:59,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:17:59,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:59,553 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:59,559 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:59,559 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/8488fab8a09d4d7e9ffd3cb9c4d8664c is 50, key is test_row_0/B:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:59,562 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211f174acb997a4610a399f4a9e5d40804_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:59,564 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211f174acb997a4610a399f4a9e5d40804_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:59,564 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f174acb997a4610a399f4a9e5d40804_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:59,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742446_1622 (size=4469) 2024-11-21T00:17:59,607 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#519 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:59,607 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3d024276f69c41459727f7090edea7eb is 175, key is test_row_0/A:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:59,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742445_1621 (size=12983) 2024-11-21T00:17:59,650 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/8488fab8a09d4d7e9ffd3cb9c4d8664c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/8488fab8a09d4d7e9ffd3cb9c4d8664c 2024-11-21T00:17:59,671 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into 8488fab8a09d4d7e9ffd3cb9c4d8664c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:59,671 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:59,671 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=13, startTime=1732148279547; duration=0sec 2024-11-21T00:17:59,671 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:17:59,671 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:17:59,671 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:17:59,678 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:17:59,678 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:17:59,678 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:59,678 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/77f9cef6c54a4abba3f17ade36ff53d9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.6 K 2024-11-21T00:17:59,679 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f9cef6c54a4abba3f17ade36ff53d9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148274903 2024-11-21T00:17:59,679 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting e318522cc7bb4800bb8f23b3bd97d9d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148275274 2024-11-21T00:17:59,680 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting a8dbc9a66df14f4d9b9b3612482ee4c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:17:59,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742447_1623 (size=31937) 2024-11-21T00:17:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:17:59,701 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:17:59,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:17:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T00:17:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:17:59,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:17:59,726 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:17:59,727 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/df22df10c0ac4819b5bd9725e00b6bbf is 50, key is test_row_0/C:col10/1732148278118/Put/seqid=0 2024-11-21T00:17:59,728 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:17:59,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:17:59,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742448_1624 (size=12983) 2024-11-21T00:17:59,771 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/df22df10c0ac4819b5bd9725e00b6bbf as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/df22df10c0ac4819b5bd9725e00b6bbf 2024-11-21T00:17:59,775 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into df22df10c0ac4819b5bd9725e00b6bbf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:17:59,775 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:17:59,775 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=13, startTime=1732148279552; duration=0sec 2024-11-21T00:17:59,775 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:17:59,775 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:17:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:17:59,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T00:17:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:17:59,893 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T00:17:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:17:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:17:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:17:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:17:59,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213442a96fd8e24ab09fb9f1dc6116ec7a_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148278146/Put/seqid=0 2024-11-21T00:17:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742449_1625 (size=12454) 2024-11-21T00:17:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:17:59,917 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213442a96fd8e24ab09fb9f1dc6116ec7a_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213442a96fd8e24ab09fb9f1dc6116ec7a_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:17:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c3f7fdda3a154f85ac4f962d5f182e41, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:17:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c3f7fdda3a154f85ac4f962d5f182e41 is 175, key is test_row_0/A:col10/1732148278146/Put/seqid=0 2024-11-21T00:17:59,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742450_1626 (size=31255) 2024-11-21T00:18:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:18:00,098 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/3d024276f69c41459727f7090edea7eb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb 2024-11-21T00:18:00,109 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 3d024276f69c41459727f7090edea7eb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:18:00,109 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:00,109 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=13, startTime=1732148279540; duration=0sec 2024-11-21T00:18:00,109 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:00,109 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:18:00,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:00,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c3f7fdda3a154f85ac4f962d5f182e41 2024-11-21T00:18:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:18:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/386ab19b383149d0a5bf65d94d94f6e4 is 50, key is test_row_0/B:col10/1732148278146/Put/seqid=0 2024-11-21T00:18:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148340350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148340353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148340353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742451_1627 (size=12301) 2024-11-21T00:18:00,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148340466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148340466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148340466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148340677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148340677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148340678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:00,793 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/386ab19b383149d0a5bf65d94d94f6e4 2024-11-21T00:18:00,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/dba6fa1eb16945e2b60faed264633120 is 50, key is test_row_0/C:col10/1732148278146/Put/seqid=0 2024-11-21T00:18:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:18:00,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742452_1628 (size=12301) 2024-11-21T00:18:00,862 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/dba6fa1eb16945e2b60faed264633120 2024-11-21T00:18:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/c3f7fdda3a154f85ac4f962d5f182e41 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41 2024-11-21T00:18:00,885 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41, entries=150, sequenceid=327, filesize=30.5 K 2024-11-21T00:18:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/386ab19b383149d0a5bf65d94d94f6e4 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4 2024-11-21T00:18:00,901 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T00:18:00,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/dba6fa1eb16945e2b60faed264633120 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120 2024-11-21T00:18:00,914 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T00:18:00,915 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d3ff459acb90743c62370493a30bdf44 in 1021ms, sequenceid=327, compaction requested=false 2024-11-21T00:18:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T00:18:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T00:18:00,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T00:18:00,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1880 sec 2024-11-21T00:18:00,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2170 sec 2024-11-21T00:18:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:00,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T00:18:01,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148280324, max timestamp=1732148280997Num uniques 859; 2024-11-21T00:18:01,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:18:01,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148280324, max timestamp=1732148280997Num uniques 859; 2024-11-21T00:18:01,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:18:01,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732148280324, max timestamp=1732148280997Num uniques 859; 2024-11-21T00:18:01,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T00:18:01,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:18:01,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:18:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:18:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a5055e749dec4327b68796fdbbec823e_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742453_1629 (size=14994) 2024-11-21T00:18:01,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:18:01,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a5055e749dec4327b68796fdbbec823e_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5055e749dec4327b68796fdbbec823e_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:01,041 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53b9ef635d6c4f69943cd4266813e801, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53b9ef635d6c4f69943cd4266813e801 is 175, key is test_row_0/A:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742454_1630 (size=39949) 2024-11-21T00:18:01,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148281031, max timestamp=1732148281066Num uniques 859; 2024-11-21T00:18:01,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:18:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148281031, max timestamp=1732148281066Num uniques 859; 2024-11-21T00:18:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:18:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148281031, max timestamp=1732148281066Num uniques 859; 2024-11-21T00:18:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732148281105, max timestamp=1732148281118Num uniques 1018; 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732148281105, max timestamp=1732148281118Num uniques 1018; 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732148281105, max timestamp=1732148281118Num uniques 1018; 2024-11-21T00:18:01,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T00:18:01,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732148281145, max timestamp=1732148281177Num uniques 1077; 2024-11-21T00:18:01,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-21T00:18:01,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732148281145, max timestamp=1732148281177Num uniques 1077; 2024-11-21T00:18:01,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-21T00:18:01,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732148281157, max timestamp=1732148281177Num uniques 1077; 2024-11-21T00:18:01,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-21T00:18:01,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148341258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148341259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732148281245, max timestamp=1732148281263Num uniques 1086; 2024-11-21T00:18:01,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-21T00:18:01,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732148281245, max timestamp=1732148281263Num uniques 1036; 2024-11-21T00:18:01,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-21T00:18:01,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732148281245, max timestamp=1732148281263Num uniques 1086; 2024-11-21T00:18:01,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732148281334, max timestamp=1732148281362Num uniques 1095; 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732148281334, max timestamp=1732148281362Num uniques 1045; 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732148281334, max timestamp=1732148281362Num uniques 1045; 2024-11-21T00:18:01,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-21T00:18:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148341401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732148281398, max timestamp=1732148281406Num uniques 1054; 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732148281398, max timestamp=1732148281406Num uniques 1054; 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732148281398, max timestamp=1732148281406Num uniques 1054; 2024-11-21T00:18:01,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-21T00:18:01,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53b9ef635d6c4f69943cd4266813e801 2024-11-21T00:18:01,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732148281433, max timestamp=1732148281463Num uniques 1063; 2024-11-21T00:18:01,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T00:18:01,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732148281433, max timestamp=1732148281463Num uniques 1063; 2024-11-21T00:18:01,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T00:18:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732148281433, max timestamp=1732148281463Num uniques 1063; 2024-11-21T00:18:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T00:18:01,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e972907614ee44f6a646cad84100d1cb is 50, key is test_row_0/B:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742455_1631 (size=12301) 2024-11-21T00:18:01,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e972907614ee44f6a646cad84100d1cb 2024-11-21T00:18:01,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/0ffdb4033d9d463aabccb96392ab2c20 is 50, key is test_row_0/C:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742456_1632 (size=12301) 2024-11-21T00:18:01,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/0ffdb4033d9d463aabccb96392ab2c20 2024-11-21T00:18:01,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/53b9ef635d6c4f69943cd4266813e801 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801 2024-11-21T00:18:01,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801, entries=200, sequenceid=350, filesize=39.0 K 2024-11-21T00:18:01,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/e972907614ee44f6a646cad84100d1cb as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb 2024-11-21T00:18:01,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T00:18:01,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/0ffdb4033d9d463aabccb96392ab2c20 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20 2024-11-21T00:18:01,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T00:18:01,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~101.34 KB/103776, currentSize=120.76 KB/123660 for d3ff459acb90743c62370493a30bdf44 in 543ms, sequenceid=350, compaction requested=true 2024-11-21T00:18:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:18:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:01,542 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:18:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:18:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:01,542 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:18:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:18:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:18:01,543 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:18:01,543 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:18:01,543 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/A is initiating minor compaction (all files) 2024-11-21T00:18:01,543 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/B is initiating minor compaction (all files) 2024-11-21T00:18:01,543 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/A in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:01,543 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/B in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:01,543 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/8488fab8a09d4d7e9ffd3cb9c4d8664c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.7 K 2024-11-21T00:18:01,543 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=100.7 K 2024-11-21T00:18:01,543 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:01,543 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. files: [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801] 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8488fab8a09d4d7e9ffd3cb9c4d8664c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d024276f69c41459727f7090edea7eb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386ab19b383149d0a5bf65d94d94f6e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732148278136 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting c3f7fdda3a154f85ac4f962d5f182e41, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732148278136 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting e972907614ee44f6a646cad84100d1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148280324 2024-11-21T00:18:01,544 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b9ef635d6c4f69943cd4266813e801, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148280324 2024-11-21T00:18:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:01,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=127.47 KB heapSize=364.69 KB 2024-11-21T00:18:01,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:18:01,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:18:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:18:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:01,550 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148341554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148341558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148341559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,569 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112192890c4fd241481cbbe3ad0eacf6b146_d3ff459acb90743c62370493a30bdf44 store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148341560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,571 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112192890c4fd241481cbbe3ad0eacf6b146_d3ff459acb90743c62370493a30bdf44, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,571 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112192890c4fd241481cbbe3ad0eacf6b146_d3ff459acb90743c62370493a30bdf44 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,579 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:18:01,579 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/257ebbd88c61421ab13c3aa52d834073 is 50, key is test_row_0/B:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b0a98d518de24bb895bb615e2253f118_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148281536/Put/seqid=0 2024-11-21T00:18:01,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742457_1633 (size=4469) 2024-11-21T00:18:01,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742458_1634 (size=13085) 2024-11-21T00:18:01,662 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/257ebbd88c61421ab13c3aa52d834073 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/257ebbd88c61421ab13c3aa52d834073 2024-11-21T00:18:01,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742459_1635 (size=14994) 2024-11-21T00:18:01,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:18:01,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b0a98d518de24bb895bb615e2253f118_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0a98d518de24bb895bb615e2253f118_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:01,675 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/B of d3ff459acb90743c62370493a30bdf44 into 257ebbd88c61421ab13c3aa52d834073(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:18:01,675 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:01,675 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/B, priority=13, startTime=1732148281542; duration=0sec 2024-11-21T00:18:01,676 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:18:01,676 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:18:01,676 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:18:01,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e26d32076db94d0698cfbe17b8ca4279, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:01,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e26d32076db94d0698cfbe17b8ca4279 is 175, key is test_row_0/A:col10/1732148281536/Put/seqid=0 2024-11-21T00:18:01,678 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:18:01,678 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1541): d3ff459acb90743c62370493a30bdf44/C is initiating minor compaction (all files) 2024-11-21T00:18:01,678 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3ff459acb90743c62370493a30bdf44/C in TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:01,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,678 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/df22df10c0ac4819b5bd9725e00b6bbf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20] into tmpdir=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp, totalSize=36.7 K 2024-11-21T00:18:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148341677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148341677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,679 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting df22df10c0ac4819b5bd9725e00b6bbf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148277480 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148341678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,679 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba6fa1eb16945e2b60faed264633120, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732148278136 2024-11-21T00:18:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148341678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148341678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,680 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ffdb4033d9d463aabccb96392ab2c20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148280324 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742460_1636 (size=39949) 2024-11-21T00:18:01,685 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:18:01,685 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/a34bce9a04e44f5c97a48b4dac76efc7 is 50, key is test_row_0/C:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:01,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742461_1637 (size=13085) 2024-11-21T00:18:01,703 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/a34bce9a04e44f5c97a48b4dac76efc7 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a34bce9a04e44f5c97a48b4dac76efc7 2024-11-21T00:18:01,706 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/C of d3ff459acb90743c62370493a30bdf44 into a34bce9a04e44f5c97a48b4dac76efc7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:18:01,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:01,706 INFO [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/C, priority=13, startTime=1732148281542; duration=0sec 2024-11-21T00:18:01,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:01,706 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:18:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:18:01,857 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:18:01,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:18:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T00:18:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:01,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:18:01,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:18:01,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:18:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148341886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148341886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148341886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148341886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148341889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:02,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:18:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:02,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,028 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3ff459acb90743c62370493a30bdf44#A#compaction#528 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:18:02,028 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/6be309a0bf9a4822a02b2700a686f245 is 175, key is test_row_0/A:col10/1732148280999/Put/seqid=0 2024-11-21T00:18:02,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742462_1638 (size=32039) 2024-11-21T00:18:02,065 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/6be309a0bf9a4822a02b2700a686f245 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6be309a0bf9a4822a02b2700a686f245 2024-11-21T00:18:02,074 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3ff459acb90743c62370493a30bdf44/A of d3ff459acb90743c62370493a30bdf44 into 6be309a0bf9a4822a02b2700a686f245(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:18:02,074 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:02,074 INFO [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., storeName=d3ff459acb90743c62370493a30bdf44/A, priority=13, startTime=1732148281542; duration=0sec 2024-11-21T00:18:02,074 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:02,074 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:18:02,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e26d32076db94d0698cfbe17b8ca4279 2024-11-21T00:18:02,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/794af4a5b9dd49cda179360d1983d96a is 50, key is test_row_0/B:col10/1732148281536/Put/seqid=0 2024-11-21T00:18:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742463_1639 (size=12301) 2024-11-21T00:18:02,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/794af4a5b9dd49cda179360d1983d96a 2024-11-21T00:18:02,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/c86bbdf5ed9c4c6780d5367062980c33 is 50, key is test_row_0/C:col10/1732148281536/Put/seqid=0 2024-11-21T00:18:02,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742464_1640 (size=12301) 2024-11-21T00:18:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:02,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:18:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148342197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148342207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148342207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148342207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148342207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:18:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:18:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:18:02,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/c86bbdf5ed9c4c6780d5367062980c33 2024-11-21T00:18:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/e26d32076db94d0698cfbe17b8ca4279 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e26d32076db94d0698cfbe17b8ca4279 2024-11-21T00:18:02,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e26d32076db94d0698cfbe17b8ca4279, entries=200, sequenceid=489, filesize=39.0 K 2024-11-21T00:18:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/794af4a5b9dd49cda179360d1983d96a as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/794af4a5b9dd49cda179360d1983d96a 2024-11-21T00:18:02,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/794af4a5b9dd49cda179360d1983d96a, entries=150, sequenceid=489, filesize=12.0 K 2024-11-21T00:18:02,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/c86bbdf5ed9c4c6780d5367062980c33 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/c86bbdf5ed9c4c6780d5367062980c33 2024-11-21T00:18:02,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/c86bbdf5ed9c4c6780d5367062980c33, entries=150, sequenceid=489, filesize=12.0 K 2024-11-21T00:18:02,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~364.64 KB/373392, currentSize=53.67 KB/54960 for d3ff459acb90743c62370493a30bdf44 in 1098ms, sequenceid=489, compaction requested=false 2024-11-21T00:18:02,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:02,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40705 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:18:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:02,646 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:18:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:02,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218d0056e979394480b7b6f8498de7dad0_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148281558/Put/seqid=0 2024-11-21T00:18:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742465_1641 (size=12454) 2024-11-21T00:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. as already flushing 2024-11-21T00:18:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148342792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148342796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148282778, max timestamp=1732148282797Num uniques 859; 2024-11-21T00:18:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:18:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732148282778, max timestamp=1732148282797Num uniques 859; 2024-11-21T00:18:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T00:18:02,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732148282778, max timestamp=1732148282797Num uniques 859; 2024-11-21T00:18:02,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5ed4808ef0e6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T00:18:02,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148342843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148342846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148342847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,908 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-21T00:18:02,909 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:02,909 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,909 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148342907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148342907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,913 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-21T00:18:02,914 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:02,914 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,914 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:02,919 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-21T00:18:02,919 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:02,919 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,920 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:02,921 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-21T00:18:02,921 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:02,921 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,922 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,922 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-21T00:18:02,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:02,922 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:02,922 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,922 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:02,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:02,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148342957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148342957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148342957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148343118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,122 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218d0056e979394480b7b6f8498de7dad0_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218d0056e979394480b7b6f8498de7dad0_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/a639425f415f4f84b9fc585e3b77e9b5, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:03,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/a639425f415f4f84b9fc585e3b77e9b5 is 175, key is test_row_0/A:col10/1732148281558/Put/seqid=0 2024-11-21T00:18:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148343126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742466_1642 (size=31255) 2024-11-21T00:18:03,145 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/a639425f415f4f84b9fc585e3b77e9b5 2024-11-21T00:18:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/a653a8b62b2d448e9e287cb0cdfbb18b is 50, key is test_row_0/B:col10/1732148281558/Put/seqid=0 2024-11-21T00:18:03,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742467_1643 (size=12301) 2024-11-21T00:18:03,166 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/a653a8b62b2d448e9e287cb0cdfbb18b 2024-11-21T00:18:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148343167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148343167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148343167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 is 50, key is test_row_0/C:col10/1732148281558/Put/seqid=0 2024-11-21T00:18:03,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742468_1644 (size=12301) 2024-11-21T00:18:03,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37772 deadline: 1732148343426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37754 deadline: 1732148343437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:18:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37738 deadline: 1732148343477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37794 deadline: 1732148343477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1732148343477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., hostname=5ed4808ef0e6,40705,1732148113864, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:18:03,586 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 2024-11-21T00:18:03,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/a639425f415f4f84b9fc585e3b77e9b5 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/a639425f415f4f84b9fc585e3b77e9b5 2024-11-21T00:18:03,592 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/a639425f415f4f84b9fc585e3b77e9b5, entries=150, sequenceid=503, filesize=30.5 K 2024-11-21T00:18:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/a653a8b62b2d448e9e287cb0cdfbb18b as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/a653a8b62b2d448e9e287cb0cdfbb18b 2024-11-21T00:18:03,595 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/a653a8b62b2d448e9e287cb0cdfbb18b, entries=150, sequenceid=503, filesize=12.0 K 2024-11-21T00:18:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 2024-11-21T00:18:03,598 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/2e2ffe2f9b3d4e0d8e4185436d9cb754, entries=150, sequenceid=503, filesize=12.0 K 2024-11-21T00:18:03,599 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for d3ff459acb90743c62370493a30bdf44 in 953ms, sequenceid=503, compaction requested=true 2024-11-21T00:18:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T00:18:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T00:18:03,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T00:18:03,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7350 sec 2024-11-21T00:18:03,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7420 sec 2024-11-21T00:18:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40705 {}] regionserver.HRegion(8855): Flush requested on d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:03,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-21T00:18:03,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:18:03,937 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-21T00:18:03,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:03,937 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:18:03,938 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:18:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T00:18:03,938 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:03,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212981d52a4f2b4444b69dbfe44675675c_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148282833/Put/seqid=0 2024-11-21T00:18:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742469_1645 (size=12454) 2024-11-21T00:18:03,947 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-21T00:18:03,947 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:03,948 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,948 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:03,988 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-21T00:18:03,988 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:03,988 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-21T00:18:03,988 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,988 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-21T00:18:03,988 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:03,988 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:18:03,988 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,988 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:03,989 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,989 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:03,989 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:03,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:18:04,007 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2349 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2330 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2279 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2330 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2316 2024-11-21T00:18:04,007 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:18:04,007 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:18:04,007 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:18:04,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:04,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:04,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:04,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:18:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:04,010 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148284010"}]},"ts":"1732148284010"} 2024-11-21T00:18:04,011 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:18:04,011 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:18:04,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:18:04,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, UNASSIGN}] 2024-11-21T00:18:04,014 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, UNASSIGN 2024-11-21T00:18:04,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=CLOSING, regionLocation=5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:04,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, UNASSIGN because future has completed 2024-11-21T00:18:04,015 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:18:04,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864}] 2024-11-21T00:18:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:04,168 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:04,168 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:18:04,168 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing d3ff459acb90743c62370493a30bdf44, disabling compactions & flushes 2024-11-21T00:18:04,168 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:04,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:18:04,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212981d52a4f2b4444b69dbfe44675675c_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212981d52a4f2b4444b69dbfe44675675c_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:04,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/787add1d92e54e62820c92ab322ce5a3, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:04,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/787add1d92e54e62820c92ab322ce5a3 is 175, key is test_row_0/A:col10/1732148282833/Put/seqid=0 2024-11-21T00:18:04,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742470_1646 (size=31255) 2024-11-21T00:18:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:04,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/787add1d92e54e62820c92ab322ce5a3 2024-11-21T00:18:04,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/b8647c30bf06497b8c1cdac6b443d412 is 50, key is test_row_0/B:col10/1732148282833/Put/seqid=0 2024-11-21T00:18:04,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742471_1647 (size=12301) 2024-11-21T00:18:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:05,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/b8647c30bf06497b8c1cdac6b443d412 2024-11-21T00:18:05,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/9283cebf8fb04e1f87a6794c12a4f769 is 50, key is test_row_0/C:col10/1732148282833/Put/seqid=0 2024-11-21T00:18:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742472_1648 (size=12301) 2024-11-21T00:18:05,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/9283cebf8fb04e1f87a6794c12a4f769 2024-11-21T00:18:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/787add1d92e54e62820c92ab322ce5a3 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/787add1d92e54e62820c92ab322ce5a3 2024-11-21T00:18:05,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/787add1d92e54e62820c92ab322ce5a3, entries=150, sequenceid=540, filesize=30.5 K 2024-11-21T00:18:05,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/b8647c30bf06497b8c1cdac6b443d412 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b8647c30bf06497b8c1cdac6b443d412 2024-11-21T00:18:05,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b8647c30bf06497b8c1cdac6b443d412, entries=150, sequenceid=540, filesize=12.0 K 2024-11-21T00:18:05,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/9283cebf8fb04e1f87a6794c12a4f769 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/9283cebf8fb04e1f87a6794c12a4f769 2024-11-21T00:18:05,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/9283cebf8fb04e1f87a6794c12a4f769, entries=150, sequenceid=540, filesize=12.0 K 2024-11-21T00:18:05,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=26.84 KB/27480 for d3ff459acb90743c62370493a30bdf44 in 1265ms, sequenceid=540, compaction requested=true 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3ff459acb90743c62370493a30bdf44: 2024-11-21T00:18:05,203 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. because compaction request was cancelled 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. after waiting 0 ms 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:A 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. because compaction request was cancelled 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3ff459acb90743c62370493a30bdf44:C, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:18:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. because compaction request was cancelled 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:B 2024-11-21T00:18:05,203 DEBUG [RS:0;5ed4808ef0e6:40705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3ff459acb90743c62370493a30bdf44:C 2024-11-21T00:18:05,203 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing d3ff459acb90743c62370493a30bdf44 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=A 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=B 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3ff459acb90743c62370493a30bdf44, store=C 2024-11-21T00:18:05,203 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T00:18:05,207 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112103f5b70e4f374944a45e730d64201a8d_d3ff459acb90743c62370493a30bdf44 is 50, key is test_row_0/A:col10/1732148283987/Put/seqid=0 2024-11-21T00:18:05,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742473_1649 (size=12454) 2024-11-21T00:18:05,611 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:18:05,615 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112103f5b70e4f374944a45e730d64201a8d_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103f5b70e4f374944a45e730d64201a8d_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:05,616 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/941cf3d23701449ca3add8c5e0f77d1c, store: [table=TestAcidGuarantees family=A region=d3ff459acb90743c62370493a30bdf44] 2024-11-21T00:18:05,617 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/941cf3d23701449ca3add8c5e0f77d1c is 175, key is test_row_0/A:col10/1732148283987/Put/seqid=0 2024-11-21T00:18:05,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742474_1650 (size=31255) 2024-11-21T00:18:06,022 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=547, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/941cf3d23701449ca3add8c5e0f77d1c 2024-11-21T00:18:06,029 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/813addcdb2f442b9b479386f1ba95b90 is 50, key is test_row_0/B:col10/1732148283987/Put/seqid=0 2024-11-21T00:18:06,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742475_1651 (size=12301) 2024-11-21T00:18:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:06,434 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/813addcdb2f442b9b479386f1ba95b90 2024-11-21T00:18:06,442 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/82f52eca93e04aec8cd6abbfdda10024 is 50, key is test_row_0/C:col10/1732148283987/Put/seqid=0 2024-11-21T00:18:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742476_1652 (size=12301) 2024-11-21T00:18:06,847 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/82f52eca93e04aec8cd6abbfdda10024 2024-11-21T00:18:06,854 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/A/941cf3d23701449ca3add8c5e0f77d1c as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/941cf3d23701449ca3add8c5e0f77d1c 2024-11-21T00:18:06,857 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/941cf3d23701449ca3add8c5e0f77d1c, entries=150, sequenceid=547, filesize=30.5 K 2024-11-21T00:18:06,858 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/B/813addcdb2f442b9b479386f1ba95b90 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/813addcdb2f442b9b479386f1ba95b90 2024-11-21T00:18:06,861 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/813addcdb2f442b9b479386f1ba95b90, entries=150, sequenceid=547, filesize=12.0 K 2024-11-21T00:18:06,862 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/.tmp/C/82f52eca93e04aec8cd6abbfdda10024 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/82f52eca93e04aec8cd6abbfdda10024 2024-11-21T00:18:06,865 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/82f52eca93e04aec8cd6abbfdda10024, entries=150, sequenceid=547, filesize=12.0 K 2024-11-21T00:18:06,865 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d3ff459acb90743c62370493a30bdf44 in 1662ms, sequenceid=547, compaction requested=true 2024-11-21T00:18:06,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801] to archive 2024-11-21T00:18:06,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:18:06,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/1222e976f48c4d4182f9324345ca9d05 2024-11-21T00:18:06,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/ab10a44382e642809a252e7b2e4d65fe 2024-11-21T00:18:06,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/fff9917607fa45d5bfc0f4dca611d1ca 2024-11-21T00:18:06,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/7f1d9191f885411eb5f2842a736714e8 2024-11-21T00:18:06,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/179a35f8b1114c50a345d8d2b692ab41 2024-11-21T00:18:06,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2287deae1ced4949ad233e1ceaa6b7e6 2024-11-21T00:18:06,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3f5a6ec82a5d48fda0a67555ce68b3ea 2024-11-21T00:18:06,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53471ca54a41462d81b8090a33499780 2024-11-21T00:18:06,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/2dc549bb017544e481d40ac3d47f31b9 2024-11-21T00:18:06,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/238f0ee52732434ab858b6f278744455 2024-11-21T00:18:06,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/142d1d5e70a14696b649138125f2af8d 2024-11-21T00:18:06,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/69784d1a237040d0880d1fd791f2235f 2024-11-21T00:18:06,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e311080224ad426d834b0f1cbc645682 2024-11-21T00:18:06,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/236ae58649df4356b2316b8a7bd419e5 2024-11-21T00:18:06,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/db001f46ac684224bde214deb0efa4d3 2024-11-21T00:18:06,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/567ca04acb5245c59ea7fe42329e134d 2024-11-21T00:18:06,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/0355f20c429543ea93299130aaffd00a 2024-11-21T00:18:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6eebfc379c29470eb88c70d3d4ce150d 2024-11-21T00:18:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/02fe41801b354d5181d882d85506e506 2024-11-21T00:18:06,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/3d024276f69c41459727f7090edea7eb 2024-11-21T00:18:06,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c64806d7fce54f7fb964792c76817f80 2024-11-21T00:18:06,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/c3f7fdda3a154f85ac4f962d5f182e41 2024-11-21T00:18:06,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/53b9ef635d6c4f69943cd4266813e801 2024-11-21T00:18:06,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b25a162472054696b0c00925cadd3924, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/754f273660ac41268231787f6b29f324, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e70578e351094cea8c5666736ae29c23, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/01d9c759dcc84d7b9ae0a96b49a2189b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/824a7c69182541dcb9c087f4e1de96c8, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/8488fab8a09d4d7e9ffd3cb9c4d8664c, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb] to archive 2024-11-21T00:18:06,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:18:06,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/4b4e3c99b4d74f239b7cd4b84538d79f 2024-11-21T00:18:06,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/49727c6bb033435296fe3ceeefd41f39 2024-11-21T00:18:06,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/24f20bb5646d4f148214c221bd9b423d 2024-11-21T00:18:06,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b25a162472054696b0c00925cadd3924 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b25a162472054696b0c00925cadd3924 2024-11-21T00:18:06,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e9ee786d604c4638bf33d4783bc2421f 2024-11-21T00:18:06,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e71a12e988e645a6bc8d7e337b06e110 2024-11-21T00:18:06,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/754f273660ac41268231787f6b29f324 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/754f273660ac41268231787f6b29f324 2024-11-21T00:18:06,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/125a4f9041854db7837a8ab9ee5561ab 2024-11-21T00:18:06,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/932497b3d1f744568e4b1f89c875a0b0 2024-11-21T00:18:06,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e70578e351094cea8c5666736ae29c23 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e70578e351094cea8c5666736ae29c23 2024-11-21T00:18:06,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e8253462b47d4b39bf72c7552e9dbd8f 2024-11-21T00:18:06,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/45e28e9ab4ed42aca260a4714b36bee9 2024-11-21T00:18:06,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/df35e800f56d43b290a625c9b73e6f98 2024-11-21T00:18:06,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/01d9c759dcc84d7b9ae0a96b49a2189b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/01d9c759dcc84d7b9ae0a96b49a2189b 2024-11-21T00:18:06,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/52f54f3530bd4ce99c05d0e3bc89feb0 2024-11-21T00:18:06,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/02a4e4286b3a48818ae73e2fc26d5797 2024-11-21T00:18:06,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/824a7c69182541dcb9c087f4e1de96c8 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/824a7c69182541dcb9c087f4e1de96c8 2024-11-21T00:18:06,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/cd404ba0889148b3816f40791bf0c08d 2024-11-21T00:18:06,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/1b74f9449954471da011be0261e6c3fd 2024-11-21T00:18:06,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/8488fab8a09d4d7e9ffd3cb9c4d8664c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/8488fab8a09d4d7e9ffd3cb9c4d8664c 2024-11-21T00:18:06,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/fbe1c68690f14d05a4f4c720669c04dd 2024-11-21T00:18:06,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/386ab19b383149d0a5bf65d94d94f6e4 2024-11-21T00:18:06,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/e972907614ee44f6a646cad84100d1cb 2024-11-21T00:18:06,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/d1694658fb6b4676b7e1d4f00be7635b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/fba1ef8378d7411386287bfa1c776e41, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/70234187923b42f9bebb29b69a90f70b, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/22dccf87bbe64faaba1318e27eacaad9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/77f9cef6c54a4abba3f17ade36ff53d9, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/df22df10c0ac4819b5bd9725e00b6bbf, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20] to archive 2024-11-21T00:18:06,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:18:06,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f64873a459dc492ca8126c0f454eb8c1 2024-11-21T00:18:06,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/25e64315db4a450eb8a5c55835de15f4 2024-11-21T00:18:06,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/280f2af600e049c99f25914a8fb0d113 2024-11-21T00:18:06,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/d1694658fb6b4676b7e1d4f00be7635b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/d1694658fb6b4676b7e1d4f00be7635b 2024-11-21T00:18:06,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/879149dff3a84ae3af29c6185dabc6fc 2024-11-21T00:18:06,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/ce25b3128bba46bfb7bfe515a1c77c81 2024-11-21T00:18:06,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/fba1ef8378d7411386287bfa1c776e41 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/fba1ef8378d7411386287bfa1c776e41 2024-11-21T00:18:06,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/eedef729b48d472bad474b8a35ba8459 2024-11-21T00:18:06,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e8af27846f6f49fd85ecd35202e56b42 2024-11-21T00:18:06,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/70234187923b42f9bebb29b69a90f70b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/70234187923b42f9bebb29b69a90f70b 2024-11-21T00:18:06,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/da5e89a548b140c481baf8a4a73e1f19 2024-11-21T00:18:06,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/bb1b7faf47da4249af3c6f4db9cae38d 2024-11-21T00:18:06,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/3a04d5bd9aed405881a4be1d83bf92ed 2024-11-21T00:18:06,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/22dccf87bbe64faaba1318e27eacaad9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/22dccf87bbe64faaba1318e27eacaad9 2024-11-21T00:18:06,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/b7b97bc9c3cd46a8b5676f976aad1f1e 2024-11-21T00:18:06,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/773b15205bb84ad5a0f247c729ae0b87 2024-11-21T00:18:06,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/77f9cef6c54a4abba3f17ade36ff53d9 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/77f9cef6c54a4abba3f17ade36ff53d9 2024-11-21T00:18:06,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/f5019c13f05a45c89f6b68e851c46a56 2024-11-21T00:18:06,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/e318522cc7bb4800bb8f23b3bd97d9d3 2024-11-21T00:18:06,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/df22df10c0ac4819b5bd9725e00b6bbf to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/df22df10c0ac4819b5bd9725e00b6bbf 2024-11-21T00:18:06,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a8dbc9a66df14f4d9b9b3612482ee4c5 2024-11-21T00:18:06,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/dba6fa1eb16945e2b60faed264633120 2024-11-21T00:18:06,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/0ffdb4033d9d463aabccb96392ab2c20 2024-11-21T00:18:06,940 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits/550.seqid, newMaxSeqId=550, maxSeqId=4 2024-11-21T00:18:06,940 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. 2024-11-21T00:18:06,940 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for d3ff459acb90743c62370493a30bdf44: Waiting for close lock at 1732148284168Running coprocessor pre-close hooks at 1732148284168Disabling compacts and flushes for region at 1732148284168Disabling writes for close at 1732148285203 (+1035 ms)Obtaining lock to block concurrent updates at 1732148285203Preparing flush snapshotting stores in d3ff459acb90743c62370493a30bdf44 at 1732148285203Finished memstore snapshotting TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732148285203Flushing stores of TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44. at 1732148285203Flushing d3ff459acb90743c62370493a30bdf44/A: creating writer at 1732148285203Flushing mob file d3ff459acb90743c62370493a30bdf44/A: appending metadata at 1732148285207 (+4 ms)Flushing mob file d3ff459acb90743c62370493a30bdf44/A: closing flushed file at 1732148285207Flushing d3ff459acb90743c62370493a30bdf44/A: appending metadata at 1732148285616 (+409 ms)Flushing d3ff459acb90743c62370493a30bdf44/A: closing flushed file at 1732148285616Flushing d3ff459acb90743c62370493a30bdf44/B: creating writer at 1732148286027 (+411 ms)Flushing d3ff459acb90743c62370493a30bdf44/B: appending metadata at 1732148286029 (+2 ms)Flushing d3ff459acb90743c62370493a30bdf44/B: closing flushed file at 1732148286029Flushing d3ff459acb90743c62370493a30bdf44/C: creating writer at 1732148286438 (+409 ms)Flushing d3ff459acb90743c62370493a30bdf44/C: appending metadata at 1732148286442 (+4 ms)Flushing d3ff459acb90743c62370493a30bdf44/C: closing flushed file at 1732148286442Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f0ace68: reopening flushed file at 1732148286853 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@541c578d: reopening flushed file at 1732148286858 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28dfbe48: reopening flushed file at 1732148286861 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d3ff459acb90743c62370493a30bdf44 in 1662ms, sequenceid=547, compaction requested=true at 1732148286865 (+4 ms)Writing region close event to WAL at 1732148286937 (+72 ms)Running coprocessor post-close hooks at 1732148286940 (+3 ms)Closed at 1732148286940 2024-11-21T00:18:06,942 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:06,942 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=d3ff459acb90743c62370493a30bdf44, regionState=CLOSED 2024-11-21T00:18:06,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 because future has completed 2024-11-21T00:18:06,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T00:18:06,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure d3ff459acb90743c62370493a30bdf44, server=5ed4808ef0e6,40705,1732148113864 in 2.9290 sec 2024-11-21T00:18:06,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-21T00:18:06,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3ff459acb90743c62370493a30bdf44, UNASSIGN in 2.9320 sec 2024-11-21T00:18:06,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T00:18:06,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9350 sec 2024-11-21T00:18:06,949 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148286949"}]},"ts":"1732148286949"} 2024-11-21T00:18:06,950 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:18:06,950 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:18:06,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9420 sec 2024-11-21T00:18:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:18:08,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:18:08,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:18:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,171 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:18:08,172 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,176 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,179 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C, FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits] 2024-11-21T00:18:08,183 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6be309a0bf9a4822a02b2700a686f245 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/6be309a0bf9a4822a02b2700a686f245 2024-11-21T00:18:08,184 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/787add1d92e54e62820c92ab322ce5a3 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/787add1d92e54e62820c92ab322ce5a3 2024-11-21T00:18:08,185 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/941cf3d23701449ca3add8c5e0f77d1c to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/941cf3d23701449ca3add8c5e0f77d1c 2024-11-21T00:18:08,186 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/a639425f415f4f84b9fc585e3b77e9b5 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/a639425f415f4f84b9fc585e3b77e9b5 2024-11-21T00:18:08,187 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e26d32076db94d0698cfbe17b8ca4279 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/A/e26d32076db94d0698cfbe17b8ca4279 2024-11-21T00:18:08,189 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/257ebbd88c61421ab13c3aa52d834073 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/257ebbd88c61421ab13c3aa52d834073 2024-11-21T00:18:08,190 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/794af4a5b9dd49cda179360d1983d96a to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/794af4a5b9dd49cda179360d1983d96a 2024-11-21T00:18:08,190 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/813addcdb2f442b9b479386f1ba95b90 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/813addcdb2f442b9b479386f1ba95b90 2024-11-21T00:18:08,191 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/a653a8b62b2d448e9e287cb0cdfbb18b to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/a653a8b62b2d448e9e287cb0cdfbb18b 2024-11-21T00:18:08,192 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b8647c30bf06497b8c1cdac6b443d412 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/B/b8647c30bf06497b8c1cdac6b443d412 2024-11-21T00:18:08,193 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/2e2ffe2f9b3d4e0d8e4185436d9cb754 2024-11-21T00:18:08,194 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/82f52eca93e04aec8cd6abbfdda10024 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/82f52eca93e04aec8cd6abbfdda10024 2024-11-21T00:18:08,195 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/9283cebf8fb04e1f87a6794c12a4f769 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/9283cebf8fb04e1f87a6794c12a4f769 2024-11-21T00:18:08,196 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a34bce9a04e44f5c97a48b4dac76efc7 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/a34bce9a04e44f5c97a48b4dac76efc7 2024-11-21T00:18:08,196 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/c86bbdf5ed9c4c6780d5367062980c33 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/C/c86bbdf5ed9c4c6780d5367062980c33 2024-11-21T00:18:08,198 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits/550.seqid to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44/recovered.edits/550.seqid 2024-11-21T00:18:08,201 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/default/TestAcidGuarantees/d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,201 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:18:08,201 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:18:08,202 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:18:08,205 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103f5b70e4f374944a45e730d64201a8d_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103f5b70e4f374944a45e730d64201a8d_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,206 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211952f266e8dd40baa66714b2fb477328_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211952f266e8dd40baa66714b2fb477328_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,207 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211bec41f4883745368354437188075e5c_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211bec41f4883745368354437188075e5c_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,208 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212981d52a4f2b4444b69dbfe44675675c_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212981d52a4f2b4444b69dbfe44675675c_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,209 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212d64d2dad5054be1b7cabede3547da5f_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212d64d2dad5054be1b7cabede3547da5f_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,210 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213442a96fd8e24ab09fb9f1dc6116ec7a_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213442a96fd8e24ab09fb9f1dc6116ec7a_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,211 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144f0495289754746adec01cfeded99b1_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144f0495289754746adec01cfeded99b1_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,212 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c8b45ea01bf4c44b21a2a42826d8ad6_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c8b45ea01bf4c44b21a2a42826d8ad6_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,213 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151681cf80ece4888b969dbd1dd2cfad9_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151681cf80ece4888b969dbd1dd2cfad9_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,213 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121827a67b9479a4b0a98101a87c3a8943a_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121827a67b9479a4b0a98101a87c3a8943a_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,214 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d1fb14baac4a54a79ba75130d8ce1e_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d1fb14baac4a54a79ba75130d8ce1e_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,215 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218d0056e979394480b7b6f8498de7dad0_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218d0056e979394480b7b6f8498de7dad0_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,216 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a001e706be3e45488fe780c76793f5ea_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a001e706be3e45488fe780c76793f5ea_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,217 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5055e749dec4327b68796fdbbec823e_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5055e749dec4327b68796fdbbec823e_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,218 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0a98d518de24bb895bb615e2253f118_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0a98d518de24bb895bb615e2253f118_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,219 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b303ef500ca94f4a8654405a247839ac_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b303ef500ca94f4a8654405a247839ac_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,219 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bce3a4bd17cf40e086c756322272e093_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bce3a4bd17cf40e086c756322272e093_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,220 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c456120f42504e2788f4a6b61f4cdb62_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c456120f42504e2788f4a6b61f4cdb62_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,221 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dafdb116dc9741d09e7d2ffdb7cd97f2_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dafdb116dc9741d09e7d2ffdb7cd97f2_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,222 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e55318d06cbf4e1daeff93a4b42ef8b5_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e55318d06cbf4e1daeff93a4b42ef8b5_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fb1f57eb9ebe4b298538f234026834ec_d3ff459acb90743c62370493a30bdf44 to hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fb1f57eb9ebe4b298538f234026834ec_d3ff459acb90743c62370493a30bdf44 2024-11-21T00:18:08,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:18:08,225 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,227 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:18:08,230 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:18:08,231 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,231 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:18:08,231 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148288231"}]},"ts":"9223372036854775807"} 2024-11-21T00:18:08,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:18:08,233 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d3ff459acb90743c62370493a30bdf44, NAME => 'TestAcidGuarantees,,1732148260795.d3ff459acb90743c62370493a30bdf44.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:18:08,233 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:18:08,234 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148288233"}]},"ts":"9223372036854775807"} 2024-11-21T00:18:08,235 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:18:08,236 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:18:08,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-21T00:18:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:18:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:18:08,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:18:08,286 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=969 (was 954) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1455 (was 1625) 2024-11-21T00:18:08,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T00:18:08,286 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:18:08,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:18:08,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:08,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:08,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:18:08,287 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T00:18:08,287 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-21T00:18:08,287 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5ed4808ef0e6,39691,1732148113019 2024-11-21T00:18:08,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T00:18:08,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:18:08,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T00:18:08,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:18:08,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T00:18:08,328 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:18:08,328 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:18:08,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:08,329 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:18:08,329 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5ed4808ef0e6,40705,1732148113864' ***** 2024-11-21T00:18:08,329 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T00:18:08,329 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:18:08,330 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T00:18:08,330 INFO [RS:0;5ed4808ef0e6:40705 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T00:18:08,330 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T00:18:08,330 INFO [RS:0;5ed4808ef0e6:40705 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(959): stopping server 5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5ed4808ef0e6:40705. 2024-11-21T00:18:08,331 DEBUG [RS:0;5ed4808ef0e6:40705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:18:08,331 DEBUG [RS:0;5ed4808ef0e6:40705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T00:18:08,331 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T00:18:08,332 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T00:18:08,332 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T00:18:08,332 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T00:18:08,332 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T00:18:08,332 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:08,332 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T00:18:08,332 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T00:18:08,332 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T00:18:08,332 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T00:18:08,341 INFO [regionserver/5ed4808ef0e6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T00:18:08,356 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/info/5bb69d19c68242fc91db3b50a4c4703f is 95, key is TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba./info:/1732148142830/DeleteFamily/seqid=0 2024-11-21T00:18:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742477_1653 (size=6514) 2024-11-21T00:18:08,532 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:08,733 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:08,760 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/info/5bb69d19c68242fc91db3b50a4c4703f 2024-11-21T00:18:08,776 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/ns/153ecc8c03874a8a87cfe4194fde415e is 93, key is TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba./ns:/1732148142766/DeleteFamily/seqid=0 2024-11-21T00:18:08,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742478_1654 (size=5906) 2024-11-21T00:18:08,871 INFO [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T00:18:08,872 INFO [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T00:18:08,933 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:09,133 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:09,180 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/ns/153ecc8c03874a8a87cfe4194fde415e 2024-11-21T00:18:09,200 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/rep_barrier/e4c5ca0ed53441eda634c231c0ee1cc0 is 102, key is TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba./rep_barrier:/1732148142766/DeleteFamily/seqid=0 2024-11-21T00:18:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742479_1655 (size=6025) 2024-11-21T00:18:09,333 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:09,533 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:09,604 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/rep_barrier/e4c5ca0ed53441eda634c231c0ee1cc0 2024-11-21T00:18:09,624 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/table/e9f162156c4548b3b32df49b60f0995e is 96, key is TestAcidGuarantees,,1732148118098.a1e0819c56248c71d5523ede3a8399ba./table:/1732148142766/DeleteFamily/seqid=0 2024-11-21T00:18:09,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742480_1656 (size=5936) 2024-11-21T00:18:09,734 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:09,934 DEBUG [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:18:10,027 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/table/e9f162156c4548b3b32df49b60f0995e 2024-11-21T00:18:10,031 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/info/5bb69d19c68242fc91db3b50a4c4703f as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/info/5bb69d19c68242fc91db3b50a4c4703f 2024-11-21T00:18:10,034 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/info/5bb69d19c68242fc91db3b50a4c4703f, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T00:18:10,035 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/ns/153ecc8c03874a8a87cfe4194fde415e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/ns/153ecc8c03874a8a87cfe4194fde415e 2024-11-21T00:18:10,038 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/ns/153ecc8c03874a8a87cfe4194fde415e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T00:18:10,039 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/rep_barrier/e4c5ca0ed53441eda634c231c0ee1cc0 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/rep_barrier/e4c5ca0ed53441eda634c231c0ee1cc0 2024-11-21T00:18:10,042 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/rep_barrier/e4c5ca0ed53441eda634c231c0ee1cc0, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T00:18:10,043 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/.tmp/table/e9f162156c4548b3b32df49b60f0995e as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/table/e9f162156c4548b3b32df49b60f0995e 2024-11-21T00:18:10,046 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/table/e9f162156c4548b3b32df49b60f0995e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T00:18:10,047 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false 2024-11-21T00:18:10,051 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T00:18:10,052 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T00:18:10,052 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T00:18:10,052 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732148288332Running coprocessor pre-close hooks at 1732148288332Disabling compacts and flushes for region at 1732148288332Disabling writes for close at 1732148288332Obtaining lock to block concurrent updates at 1732148288332Preparing flush snapshotting stores in 1588230740 at 1732148288332Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732148288333 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732148288333Flushing 1588230740/info: creating writer at 1732148288334 (+1 ms)Flushing 1588230740/info: appending metadata at 1732148288355 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732148288356 (+1 ms)Flushing 1588230740/ns: creating writer at 1732148288763 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732148288776 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732148288776Flushing 1588230740/rep_barrier: creating writer at 1732148289182 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732148289200 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732148289200Flushing 1588230740/table: creating writer at 1732148289609 (+409 ms)Flushing 1588230740/table: appending metadata at 1732148289623 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732148289623Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@420d277a: reopening flushed file at 1732148290031 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c600f3d: reopening flushed file at 1732148290035 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29538ff4: reopening flushed file at 1732148290039 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7550c564: reopening flushed file at 1732148290042 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false at 1732148290047 (+5 ms)Writing region close event to WAL at 1732148290048 (+1 ms)Running coprocessor post-close hooks at 1732148290052 (+4 ms)Closed at 1732148290052 2024-11-21T00:18:10,052 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T00:18:10,134 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(976): stopping server 5ed4808ef0e6,40705,1732148113864; all regions closed. 2024-11-21T00:18:10,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741834_1010 (size=25344) 2024-11-21T00:18:10,141 DEBUG [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/oldWALs 2024-11-21T00:18:10,142 INFO [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5ed4808ef0e6%2C40705%2C1732148113864.meta:.meta(num 1732148117592) 2024-11-21T00:18:10,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741832_1008 (size=18259696) 2024-11-21T00:18:10,146 DEBUG [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/oldWALs 2024-11-21T00:18:10,146 INFO [RS:0;5ed4808ef0e6:40705 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5ed4808ef0e6%2C40705%2C1732148113864:(num 1732148117039) 2024-11-21T00:18:10,146 DEBUG [RS:0;5ed4808ef0e6:40705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:18:10,146 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T00:18:10,146 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T00:18:10,146 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.ChoreService(370): Chore service for: regionserver/5ed4808ef0e6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T00:18:10,146 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T00:18:10,146 INFO [regionserver/5ed4808ef0e6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T00:18:10,148 INFO [RS:0;5ed4808ef0e6:40705 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40705 2024-11-21T00:18:10,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T00:18:10,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5ed4808ef0e6,40705,1732148113864 2024-11-21T00:18:10,162 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T00:18:10,172 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5ed4808ef0e6,40705,1732148113864] 2024-11-21T00:18:10,183 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5ed4808ef0e6,40705,1732148113864 already deleted, retry=false 2024-11-21T00:18:10,183 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5ed4808ef0e6,40705,1732148113864 expired; onlineServers=0 2024-11-21T00:18:10,183 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5ed4808ef0e6,39691,1732148113019' ***** 2024-11-21T00:18:10,183 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T00:18:10,183 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T00:18:10,184 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T00:18:10,184 DEBUG [M:0;5ed4808ef0e6:39691 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T00:18:10,184 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T00:18:10,184 DEBUG [M:0;5ed4808ef0e6:39691 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T00:18:10,184 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148116642 {}] cleaner.HFileCleaner(306): Exit Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148116642,5,FailOnTimeoutGroup] 2024-11-21T00:18:10,184 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148116650 {}] cleaner.HFileCleaner(306): Exit Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148116650,5,FailOnTimeoutGroup] 2024-11-21T00:18:10,184 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.ChoreService(370): Chore service for: master/5ed4808ef0e6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T00:18:10,185 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T00:18:10,185 DEBUG [M:0;5ed4808ef0e6:39691 {}] master.HMaster(1795): Stopping service threads 2024-11-21T00:18:10,185 INFO [M:0;5ed4808ef0e6:39691 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T00:18:10,185 INFO [M:0;5ed4808ef0e6:39691 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T00:18:10,185 ERROR [M:0;5ed4808ef0e6:39691 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:36975 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36975,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T00:18:10,186 INFO [M:0;5ed4808ef0e6:39691 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T00:18:10,186 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T00:18:10,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T00:18:10,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:18:10,193 DEBUG [M:0;5ed4808ef0e6:39691 {}] zookeeper.ZKUtil(347): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T00:18:10,193 WARN [M:0;5ed4808ef0e6:39691 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T00:18:10,194 INFO [M:0;5ed4808ef0e6:39691 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/.lastflushedseqids 2024-11-21T00:18:10,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742481_1657 (size=72) 2024-11-21T00:18:10,206 INFO [M:0;5ed4808ef0e6:39691 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T00:18:10,206 INFO [M:0;5ed4808ef0e6:39691 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T00:18:10,207 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T00:18:10,207 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:18:10,207 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:18:10,207 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T00:18:10,207 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:18:10,207 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=782.50 KB heapSize=958.98 KB 2024-11-21T00:18:10,222 DEBUG [M:0;5ed4808ef0e6:39691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/65d2af8e3a254e72b3ed1a9eb134ebc9 is 82, key is hbase:meta,,1/info:regioninfo/1732148117699/Put/seqid=0 2024-11-21T00:18:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742482_1658 (size=5672) 2024-11-21T00:18:10,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:18:10,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40705-0x1015abea6c30001, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:18:10,273 INFO [RS:0;5ed4808ef0e6:40705 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T00:18:10,273 INFO [RS:0;5ed4808ef0e6:40705 {}] regionserver.HRegionServer(1031): Exiting; stopping=5ed4808ef0e6,40705,1732148113864; zookeeper connection closed. 2024-11-21T00:18:10,274 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2cd899a6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2cd899a6 2024-11-21T00:18:10,274 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T00:18:10,631 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/65d2af8e3a254e72b3ed1a9eb134ebc9 2024-11-21T00:18:10,657 DEBUG [M:0;5ed4808ef0e6:39691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a8fab17b7fe428b8f78d98ed9556705 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1732148262870/Put/seqid=0 2024-11-21T00:18:10,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742483_1659 (size=44723) 2024-11-21T00:18:11,061 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=781.94 KB at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a8fab17b7fe428b8f78d98ed9556705 2024-11-21T00:18:11,064 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3a8fab17b7fe428b8f78d98ed9556705 2024-11-21T00:18:11,078 DEBUG [M:0;5ed4808ef0e6:39691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2e5a9ab0aac497bb5bade327b661793 is 69, key is 5ed4808ef0e6,40705,1732148113864/rs:state/1732148116778/Put/seqid=0 2024-11-21T00:18:11,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073742484_1660 (size=5156) 2024-11-21T00:18:11,482 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2e5a9ab0aac497bb5bade327b661793 2024-11-21T00:18:11,486 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/65d2af8e3a254e72b3ed1a9eb134ebc9 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/65d2af8e3a254e72b3ed1a9eb134ebc9 2024-11-21T00:18:11,489 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/65d2af8e3a254e72b3ed1a9eb134ebc9, entries=8, sequenceid=2185, filesize=5.5 K 2024-11-21T00:18:11,489 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a8fab17b7fe428b8f78d98ed9556705 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a8fab17b7fe428b8f78d98ed9556705 2024-11-21T00:18:11,492 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3a8fab17b7fe428b8f78d98ed9556705 2024-11-21T00:18:11,492 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a8fab17b7fe428b8f78d98ed9556705, entries=166, sequenceid=2185, filesize=43.7 K 2024-11-21T00:18:11,493 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2e5a9ab0aac497bb5bade327b661793 as hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d2e5a9ab0aac497bb5bade327b661793 2024-11-21T00:18:11,496 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36975/user/jenkins/test-data/62fe42a9-17c8-db37-de12-b1928635db2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d2e5a9ab0aac497bb5bade327b661793, entries=1, sequenceid=2185, filesize=5.0 K 2024-11-21T00:18:11,497 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(3140): Finished flush of dataSize ~782.50 KB/801276, heapSize ~958.68 KB/981688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2185, compaction requested=false 2024-11-21T00:18:11,498 INFO [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:18:11,498 DEBUG [M:0;5ed4808ef0e6:39691 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732148290206Disabling compacts and flushes for region at 1732148290206Disabling writes for close at 1732148290207 (+1 ms)Obtaining lock to block concurrent updates at 1732148290207Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732148290207Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=801276, getHeapSize=981928, getOffHeapSize=0, getCellsCount=2376 at 1732148290208 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732148290208Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732148290208Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732148290222 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732148290222Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732148290638 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732148290657 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732148290657Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732148291064 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732148291078 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732148291078Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ffdb7ae: reopening flushed file at 1732148291485 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d93f811: reopening flushed file at 1732148291489 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fe5b8d4: reopening flushed file at 1732148291492 (+3 ms)Finished flush of dataSize ~782.50 KB/801276, heapSize ~958.68 KB/981688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2185, compaction requested=false at 1732148291497 (+5 ms)Writing region close event to WAL at 1732148291498 (+1 ms)Closed at 1732148291498 2024-11-21T00:18:11,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45765 is added to blk_1073741830_1006 (size=944484) 2024-11-21T00:18:11,501 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T00:18:11,501 INFO [M:0;5ed4808ef0e6:39691 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T00:18:11,501 INFO [M:0;5ed4808ef0e6:39691 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39691 2024-11-21T00:18:11,501 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T00:18:11,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:18:11,651 INFO [M:0;5ed4808ef0e6:39691 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T00:18:11,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39691-0x1015abea6c30000, quorum=127.0.0.1:54550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:18:11,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T00:18:11,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T00:18:11,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T00:18:11,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T00:18:11,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.log.dir/,STOPPED} 2024-11-21T00:18:11,664 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T00:18:11,664 WARN [BP-1098900603-172.17.0.2-1732148108301 heartbeating to localhost/127.0.0.1:36975 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T00:18:11,664 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T00:18:11,664 WARN [BP-1098900603-172.17.0.2-1732148108301 heartbeating to localhost/127.0.0.1:36975 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1098900603-172.17.0.2-1732148108301 (Datanode Uuid 7e3c78df-4fbd-461e-beb8-621a666ccb9b) service to localhost/127.0.0.1:36975 2024-11-21T00:18:11,668 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data1/current/BP-1098900603-172.17.0.2-1732148108301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T00:18:11,669 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/cluster_a06f4331-8856-a1a7-7441-48ae907027c2/data/data2/current/BP-1098900603-172.17.0.2-1732148108301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T00:18:11,670 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T00:18:11,677 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T00:18:11,678 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T00:18:11,678 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T00:18:11,678 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T00:18:11,679 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6a36fd79-2699-fcc3-817c-d3d77c3102e4/hadoop.log.dir/,STOPPED} 2024-11-21T00:18:11,695 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T00:18:11,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down